summaryrefslogtreecommitdiff
path: root/drivers/firewire/core-cdev.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/firewire/core-cdev.c')
-rw-r--r--drivers/firewire/core-cdev.c770
1 files changed, 443 insertions, 327 deletions
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c
index 9c89f7d53e99..49dc1612c691 100644
--- a/drivers/firewire/core-cdev.c
+++ b/drivers/firewire/core-cdev.c
@@ -14,7 +14,6 @@
#include <linux/errno.h>
#include <linux/firewire.h>
#include <linux/firewire-cdev.h>
-#include <linux/idr.h>
#include <linux/irqflags.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
@@ -35,14 +34,21 @@
#include "core.h"
+#include <trace/events/firewire.h>
+
+#include "packet-header-definitions.h"
/*
* ABI version history is documented in linux/firewire-cdev.h.
*/
-#define FW_CDEV_KERNEL_VERSION 5
+#define FW_CDEV_KERNEL_VERSION 6
#define FW_CDEV_VERSION_EVENT_REQUEST2 4
#define FW_CDEV_VERSION_ALLOCATE_REGION_END 4
#define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 5
+#define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP 6
+
+static DEFINE_SPINLOCK(phy_receiver_list_lock);
+static LIST_HEAD(phy_receiver_list);
struct client {
u32 version;
@@ -50,7 +56,7 @@ struct client {
spinlock_t lock;
bool in_shutdown;
- struct idr resource_idr;
+ struct xarray resource_xa;
struct list_head event_list;
wait_queue_head_t wait;
wait_queue_head_t tx_flush_wait;
@@ -111,6 +117,7 @@ struct inbound_transaction_resource {
struct client_resource resource;
struct fw_card *card;
struct fw_request *request;
+ bool is_fcp;
void *data;
size_t length;
};
@@ -134,8 +141,41 @@ struct iso_resource {
struct iso_resource_event *e_alloc, *e_dealloc;
};
+static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource)
+{
+ return container_of(resource, struct address_handler_resource, resource);
+}
+
+static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource)
+{
+ return container_of(resource, struct inbound_transaction_resource, resource);
+}
+
+static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource)
+{
+ return container_of(resource, struct descriptor_resource, resource);
+}
+
+static struct iso_resource *to_iso_resource(struct client_resource *resource)
+{
+ return container_of(resource, struct iso_resource, resource);
+}
+
static void release_iso_resource(struct client *, struct client_resource *);
+static int is_iso_resource(const struct client_resource *resource)
+{
+ return resource->release == release_iso_resource;
+}
+
+static void release_transaction(struct client *client,
+ struct client_resource *resource);
+
+static int is_outbound_transaction_resource(const struct client_resource *resource)
+{
+ return resource->release == release_transaction;
+}
+
static void schedule_iso_resource(struct iso_resource *r, unsigned long delay)
{
client_get(r->client);
@@ -143,13 +183,6 @@ static void schedule_iso_resource(struct iso_resource *r, unsigned long delay)
client_put(r->client);
}
-static void schedule_if_iso_resource(struct client_resource *resource)
-{
- if (resource->release == release_iso_resource)
- schedule_iso_resource(container_of(resource,
- struct iso_resource, resource), 0);
-}
-
/*
* dequeue_event() just kfree()'s the event, so the event has to be
* the first field in a struct XYZ_event.
@@ -168,7 +201,10 @@ struct outbound_transaction_event {
struct event event;
struct client *client;
struct outbound_transaction_resource r;
- struct fw_cdev_event_response response;
+ union {
+ struct fw_cdev_event_response without_tstamp;
+ struct fw_cdev_event_response2 with_tstamp;
+ } rsp;
};
struct inbound_transaction_event {
@@ -176,6 +212,7 @@ struct inbound_transaction_event {
union {
struct fw_cdev_event_request request;
struct fw_cdev_event_request2 request2;
+ struct fw_cdev_event_request3 with_tstamp;
} req;
};
@@ -198,12 +235,18 @@ struct outbound_phy_packet_event {
struct event event;
struct client *client;
struct fw_packet p;
- struct fw_cdev_event_phy_packet phy_packet;
+ union {
+ struct fw_cdev_event_phy_packet without_tstamp;
+ struct fw_cdev_event_phy_packet2 with_tstamp;
+ } phy_packet;
};
struct inbound_phy_packet_event {
struct event event;
- struct fw_cdev_event_phy_packet phy_packet;
+ union {
+ struct fw_cdev_event_phy_packet without_tstamp;
+ struct fw_cdev_event_phy_packet2 with_tstamp;
+ } phy_packet;
};
#ifdef CONFIG_COMPAT
@@ -256,7 +299,7 @@ static int fw_device_op_open(struct inode *inode, struct file *file)
client->device = device;
spin_lock_init(&client->lock);
- idr_init(&client->resource_idr);
+ xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);
INIT_LIST_HEAD(&client->event_list);
init_waitqueue_head(&client->wait);
init_waitqueue_head(&client->tx_flush_wait);
@@ -272,19 +315,17 @@ static int fw_device_op_open(struct inode *inode, struct file *file)
static void queue_event(struct client *client, struct event *event,
void *data0, size_t size0, void *data1, size_t size1)
{
- unsigned long flags;
-
event->v[0].data = data0;
event->v[0].size = size0;
event->v[1].data = data1;
event->v[1].size = size1;
- spin_lock_irqsave(&client->lock, flags);
- if (client->in_shutdown)
- kfree(event);
- else
- list_add_tail(&event->link, &client->event_list);
- spin_unlock_irqrestore(&client->lock, flags);
+ scoped_guard(spinlock_irqsave, &client->lock) {
+ if (client->in_shutdown)
+ kfree(event);
+ else
+ list_add_tail(&event->link, &client->event_list);
+ }
wake_up_interruptible(&client->wait);
}
@@ -306,10 +347,10 @@ static int dequeue_event(struct client *client,
fw_device_is_shutdown(client->device))
return -ENODEV;
- spin_lock_irq(&client->lock);
- event = list_first_entry(&client->event_list, struct event, link);
- list_del(&event->link);
- spin_unlock_irq(&client->lock);
+ scoped_guard(spinlock_irq, &client->lock) {
+ event = list_first_entry(&client->event_list, struct event, link);
+ list_del(&event->link);
+ }
total = 0;
for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) {
@@ -341,7 +382,7 @@ static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,
{
struct fw_card *card = client->device->card;
- spin_lock_irq(&card->lock);
+ guard(spinlock_irq)(&card->lock);
event->closure = client->bus_reset_closure;
event->type = FW_CDEV_EVENT_BUS_RESET;
@@ -351,8 +392,6 @@ static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,
event->bm_node_id = card->bm_node_id;
event->irm_node_id = card->irm_node->node_id;
event->root_node_id = card->root_node->node_id;
-
- spin_unlock_irq(&card->lock);
}
static void for_each_client(struct fw_device *device,
@@ -360,22 +399,17 @@ static void for_each_client(struct fw_device *device,
{
struct client *c;
- mutex_lock(&device->client_list_mutex);
+ guard(mutex)(&device->client_list_mutex);
+
list_for_each_entry(c, &device->client_list, link)
callback(c);
- mutex_unlock(&device->client_list_mutex);
-}
-
-static int schedule_reallocations(int id, void *p, void *data)
-{
- schedule_if_iso_resource(p);
-
- return 0;
}
static void queue_bus_reset_event(struct client *client)
{
struct bus_reset_event *e;
+ struct client_resource *resource;
+ unsigned long index;
e = kzalloc(sizeof(*e), GFP_KERNEL);
if (e == NULL)
@@ -386,9 +420,12 @@ static void queue_bus_reset_event(struct client *client)
queue_event(client, &e->event,
&e->reset, sizeof(e->reset), NULL, 0);
- spin_lock_irq(&client->lock);
- idr_for_each(&client->resource_idr, schedule_reallocations, client);
- spin_unlock_irq(&client->lock);
+ guard(spinlock_irq)(&client->lock);
+
+ xa_for_each(&client->resource_xa, index, resource) {
+ if (is_iso_resource(resource))
+ schedule_iso_resource(to_iso_resource(resource), 0);
+ }
}
void fw_device_cdev_update(struct fw_device *device)
@@ -439,23 +476,20 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
a->version = FW_CDEV_KERNEL_VERSION;
a->card = client->device->card->index;
- down_read(&fw_device_rwsem);
+ scoped_guard(rwsem_read, &fw_device_rwsem) {
+ if (a->rom != 0) {
+ size_t want = a->rom_length;
+ size_t have = client->device->config_rom_length * 4;
- if (a->rom != 0) {
- size_t want = a->rom_length;
- size_t have = client->device->config_rom_length * 4;
-
- ret = copy_to_user(u64_to_uptr(a->rom),
- client->device->config_rom, min(want, have));
+ ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom,
+ min(want, have));
+ if (ret != 0)
+ return -EFAULT;
+ }
+ a->rom_length = client->device->config_rom_length * 4;
}
- a->rom_length = client->device->config_rom_length * 4;
-
- up_read(&fw_device_rwsem);
- if (ret != 0)
- return -EFAULT;
-
- mutex_lock(&client->device->client_list_mutex);
+ guard(mutex)(&client->device->client_list_mutex);
client->bus_reset_closure = a->bus_reset_closure;
if (a->bus_reset != 0) {
@@ -466,37 +500,36 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
if (ret == 0 && list_empty(&client->link))
list_add_tail(&client->link, &client->device->client_list);
- mutex_unlock(&client->device->client_list_mutex);
-
return ret ? -EFAULT : 0;
}
-static int add_client_resource(struct client *client,
- struct client_resource *resource, gfp_t gfp_mask)
+static int add_client_resource(struct client *client, struct client_resource *resource,
+ gfp_t gfp_mask)
{
- bool preload = gfpflags_allow_blocking(gfp_mask);
- unsigned long flags;
int ret;
- if (preload)
- idr_preload(gfp_mask);
- spin_lock_irqsave(&client->lock, flags);
-
- if (client->in_shutdown)
- ret = -ECANCELED;
- else
- ret = idr_alloc(&client->resource_idr, resource, 0, 0,
- GFP_NOWAIT);
- if (ret >= 0) {
- resource->handle = ret;
- client_get(client);
- schedule_if_iso_resource(resource);
+ scoped_guard(spinlock_irqsave, &client->lock) {
+ u32 index;
+
+ if (client->in_shutdown) {
+ ret = -ECANCELED;
+ } else {
+ if (gfpflags_allow_blocking(gfp_mask)) {
+ ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,
+ GFP_NOWAIT);
+ } else {
+ ret = xa_alloc_bh(&client->resource_xa, &index, resource,
+ xa_limit_32b, GFP_NOWAIT);
+ }
+ }
+ if (ret >= 0) {
+ resource->handle = index;
+ client_get(client);
+ if (is_iso_resource(resource))
+ schedule_iso_resource(to_iso_resource(resource), 0);
+ }
}
- spin_unlock_irqrestore(&client->lock, flags);
- if (preload)
- idr_preload_end();
-
return ret < 0 ? ret : 0;
}
@@ -504,19 +537,19 @@ static int release_client_resource(struct client *client, u32 handle,
client_resource_release_fn_t release,
struct client_resource **return_resource)
{
+ unsigned long index = handle;
struct client_resource *resource;
- spin_lock_irq(&client->lock);
- if (client->in_shutdown)
- resource = NULL;
- else
- resource = idr_find(&client->resource_idr, handle);
- if (resource && resource->release == release)
- idr_remove(&client->resource_idr, handle);
- spin_unlock_irq(&client->lock);
+ scoped_guard(spinlock_irq, &client->lock) {
+ if (client->in_shutdown)
+ return -EINVAL;
- if (!(resource && resource->release == release))
- return -EINVAL;
+ resource = xa_load(&client->resource_xa, index);
+ if (!resource || resource->release != release)
+ return -EINVAL;
+
+ xa_erase(&client->resource_xa, handle);
+ }
if (return_resource)
*return_resource = resource;
@@ -533,43 +566,66 @@ static void release_transaction(struct client *client,
{
}
-static void complete_transaction(struct fw_card *card, int rcode,
- void *payload, size_t length, void *data)
+static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp,
+ u32 response_tstamp, void *payload, size_t length, void *data)
{
struct outbound_transaction_event *e = data;
- struct fw_cdev_event_response *rsp = &e->response;
struct client *client = e->client;
- unsigned long flags;
+ unsigned long index = e->r.resource.handle;
- if (length < rsp->length)
- rsp->length = length;
- if (rcode == RCODE_COMPLETE)
- memcpy(rsp->data, payload, rsp->length);
+ scoped_guard(spinlock_irqsave, &client->lock) {
+ xa_erase(&client->resource_xa, index);
+ if (client->in_shutdown)
+ wake_up(&client->tx_flush_wait);
+ }
- spin_lock_irqsave(&client->lock, flags);
- idr_remove(&client->resource_idr, e->r.resource.handle);
- if (client->in_shutdown)
- wake_up(&client->tx_flush_wait);
- spin_unlock_irqrestore(&client->lock, flags);
+ switch (e->rsp.without_tstamp.type) {
+ case FW_CDEV_EVENT_RESPONSE:
+ {
+ struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
+
+ if (length < rsp->length)
+ rsp->length = length;
+ if (rcode == RCODE_COMPLETE)
+ memcpy(rsp->data, payload, rsp->length);
+
+ rsp->rcode = rcode;
+
+ // In the case that sizeof(*rsp) doesn't align with the position of the
+ // data, and the read is short, preserve an extra copy of the data
+ // to stay compatible with a pre-2.6.27 bug. Since the bug is harmless
+ // for short reads and some apps depended on it, this is both safe
+ // and prudent for compatibility.
+ if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data))
+ queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length);
+ else
+ queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
- rsp->type = FW_CDEV_EVENT_RESPONSE;
- rsp->rcode = rcode;
+ break;
+ }
+ case FW_CDEV_EVENT_RESPONSE2:
+ {
+ struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
- /*
- * In the case that sizeof(*rsp) doesn't align with the position of the
- * data, and the read is short, preserve an extra copy of the data
- * to stay compatible with a pre-2.6.27 bug. Since the bug is harmless
- * for short reads and some apps depended on it, this is both safe
- * and prudent for compatibility.
- */
- if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data))
- queue_event(client, &e->event, rsp, sizeof(*rsp),
- rsp->data, rsp->length);
- else
- queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length,
- NULL, 0);
+ if (length < rsp->length)
+ rsp->length = length;
+ if (rcode == RCODE_COMPLETE)
+ memcpy(rsp->data, payload, rsp->length);
+
+ rsp->rcode = rcode;
+ rsp->request_tstamp = request_tstamp;
+ rsp->response_tstamp = response_tstamp;
- /* Drop the idr's reference */
+ queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
+
+ break;
+ }
+ default:
+ WARN_ON(1);
+ break;
+ }
+
+ // Drop the xarray's reference.
client_put(client);
}
@@ -578,6 +634,7 @@ static int init_request(struct client *client,
int destination_id, int speed)
{
struct outbound_transaction_event *e;
+ void *payload;
int ret;
if (request->tcode != TCODE_STREAM_DATA &&
@@ -591,14 +648,25 @@ static int init_request(struct client *client,
e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL);
if (e == NULL)
return -ENOMEM;
-
e->client = client;
- e->response.length = request->length;
- e->response.closure = request->closure;
- if (request->data &&
- copy_from_user(e->response.data,
- u64_to_uptr(request->data), request->length)) {
+ if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
+ struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
+
+ rsp->type = FW_CDEV_EVENT_RESPONSE;
+ rsp->length = request->length;
+ rsp->closure = request->closure;
+ payload = rsp->data;
+ } else {
+ struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
+
+ rsp->type = FW_CDEV_EVENT_RESPONSE2;
+ rsp->length = request->length;
+ rsp->closure = request->closure;
+ payload = rsp->data;
+ }
+
+ if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) {
ret = -EFAULT;
goto failed;
}
@@ -608,10 +676,9 @@ static int init_request(struct client *client,
if (ret < 0)
goto failed;
- fw_send_request(client->device->card, &e->r.transaction,
- request->tcode, destination_id, request->generation,
- speed, request->offset, e->response.data,
- request->length, complete_transaction, e);
+ fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode,
+ destination_id, request->generation, speed, request->offset,
+ payload, request->length, complete_transaction, e);
return 0;
failed:
@@ -643,19 +710,13 @@ static int ioctl_send_request(struct client *client, union ioctl_arg *arg)
client->device->max_speed);
}
-static inline bool is_fcp_request(struct fw_request *request)
-{
- return request == NULL;
-}
-
static void release_request(struct client *client,
struct client_resource *resource)
{
- struct inbound_transaction_resource *r = container_of(resource,
- struct inbound_transaction_resource, resource);
+ struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource);
- if (is_fcp_request(r->request))
- kfree(r->data);
+ if (r->is_fcp)
+ fw_request_put(r->request);
else
fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR);
@@ -669,15 +730,20 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
void *payload, size_t length, void *callback_data)
{
struct address_handler_resource *handler = callback_data;
+ bool is_fcp = is_in_fcp_region(offset, length);
struct inbound_transaction_resource *r;
struct inbound_transaction_event *e;
size_t event_size0;
- void *fcp_frame = NULL;
int ret;
/* card may be different from handler->client->device->card */
fw_card_get(card);
+ // Extend the lifetime of data for request so that its payload is safely accessible in
+ // the process context for the client.
+ if (is_fcp)
+ fw_request_get(request);
+
r = kmalloc(sizeof(*r), GFP_ATOMIC);
e = kmalloc(sizeof(*e), GFP_ATOMIC);
if (r == NULL || e == NULL)
@@ -685,21 +751,10 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
r->card = card;
r->request = request;
+ r->is_fcp = is_fcp;
r->data = payload;
r->length = length;
- if (is_fcp_request(request)) {
- /*
- * FIXME: Let core-transaction.c manage a
- * single reference-counted copy?
- */
- fcp_frame = kmemdup(payload, length, GFP_ATOMIC);
- if (fcp_frame == NULL)
- goto failed;
-
- r->data = fcp_frame;
- }
-
r->resource.release = release_request;
ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC);
if (ret < 0)
@@ -718,7 +773,7 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
req->handle = r->resource.handle;
req->closure = handler->closure;
event_size0 = sizeof(*req);
- } else {
+ } else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
struct fw_cdev_event_request2 *req = &e->req.request2;
req->type = FW_CDEV_EVENT_REQUEST2;
@@ -732,6 +787,21 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
req->handle = r->resource.handle;
req->closure = handler->closure;
event_size0 = sizeof(*req);
+ } else {
+ struct fw_cdev_event_request3 *req = &e->req.with_tstamp;
+
+ req->type = FW_CDEV_EVENT_REQUEST3;
+ req->tcode = tcode;
+ req->offset = offset;
+ req->source_node_id = source;
+ req->destination_node_id = destination;
+ req->card = card->index;
+ req->generation = generation;
+ req->length = length;
+ req->handle = r->resource.handle;
+ req->closure = handler->closure;
+ req->tstamp = fw_request_get_timestamp(request);
+ event_size0 = sizeof(*req);
}
queue_event(handler->client, &e->event,
@@ -741,10 +811,11 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
failed:
kfree(r);
kfree(e);
- kfree(fcp_frame);
- if (!is_fcp_request(request))
+ if (!is_fcp)
fw_send_response(card, request, RCODE_CONFLICT_ERROR);
+ else
+ fw_request_put(request);
fw_card_put(card);
}
@@ -752,8 +823,7 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
static void release_address_handler(struct client *client,
struct client_resource *resource)
{
- struct address_handler_resource *r =
- container_of(resource, struct address_handler_resource, resource);
+ struct address_handler_resource *r = to_address_handler_resource(resource);
fw_core_remove_address_handler(&r->handler);
kfree(r);
@@ -817,19 +887,20 @@ static int ioctl_send_response(struct client *client, union ioctl_arg *arg)
release_request, &resource) < 0)
return -EINVAL;
- r = container_of(resource, struct inbound_transaction_resource,
- resource);
- if (is_fcp_request(r->request))
+ r = to_inbound_transaction_resource(resource);
+ if (r->is_fcp) {
+ fw_request_put(r->request);
goto out;
+ }
if (a->length != fw_get_response_length(r->request)) {
ret = -EINVAL;
- kfree(r->request);
+ fw_request_put(r->request);
goto out;
}
if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) {
ret = -EFAULT;
- kfree(r->request);
+ fw_request_put(r->request);
goto out;
}
fw_send_response(r->card, r->request, a->rcode);
@@ -850,8 +921,7 @@ static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg)
static void release_descriptor(struct client *client,
struct client_resource *resource)
{
- struct descriptor_resource *r =
- container_of(resource, struct descriptor_resource, resource);
+ struct descriptor_resource *r = to_descriptor_resource(resource);
fw_core_remove_descriptor(&r->descriptor);
kfree(r);
@@ -870,11 +940,12 @@ static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg)
if (a->length > 256)
return -EINVAL;
- r = kmalloc(sizeof(*r) + a->length * 4, GFP_KERNEL);
+ r = kmalloc(struct_size(r, data, a->length), GFP_KERNEL);
if (r == NULL)
return -ENOMEM;
- if (copy_from_user(r->data, u64_to_uptr(a->data), a->length * 4)) {
+ if (copy_from_user(r->data, u64_to_uptr(a->data),
+ flex_array_size(r, data, a->length))) {
ret = -EFAULT;
goto failed;
}
@@ -915,7 +986,7 @@ static void iso_callback(struct fw_iso_context *context, u32 cycle,
struct client *client = data;
struct iso_interrupt_event *e;
- e = kmalloc(sizeof(*e) + header_length, GFP_ATOMIC);
+ e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL);
if (e == NULL)
return;
@@ -934,7 +1005,7 @@ static void iso_mc_callback(struct fw_iso_context *context,
struct client *client = data;
struct iso_interrupt_mc_event *e;
- e = kmalloc(sizeof(*e), GFP_ATOMIC);
+ e = kmalloc(sizeof(*e), GFP_KERNEL);
if (e == NULL)
return;
@@ -1016,10 +1087,10 @@ static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)
if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW)
context->drop_overflow_headers = true;
- /* We only support one context at this time. */
- spin_lock_irq(&client->lock);
+ // We only support one context at this time.
+ guard(spinlock_irq)(&client->lock);
+
if (client->iso_context != NULL) {
- spin_unlock_irq(&client->lock);
fw_iso_context_destroy(context);
return -EBUSY;
@@ -1029,7 +1100,6 @@ static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)
client->device->card,
iso_dma_direction(context));
if (ret < 0) {
- spin_unlock_irq(&client->lock);
fw_iso_context_destroy(context);
return ret;
@@ -1038,7 +1108,6 @@ static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)
}
client->iso_closure = a->closure;
client->iso_context = context;
- spin_unlock_irq(&client->lock);
a->handle = 0;
@@ -1072,10 +1141,7 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)
unsigned long payload, buffer_end, transmit_header_bytes = 0;
u32 control;
int count;
- struct {
- struct fw_iso_packet packet;
- u8 header[256];
- } u;
+ DEFINE_RAW_FLEX(struct fw_iso_packet, u, header, 64);
if (ctx == NULL || a->handle != 0)
return -EINVAL;
@@ -1107,29 +1173,29 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)
while (p < end) {
if (get_user(control, &p->control))
return -EFAULT;
- u.packet.payload_length = GET_PAYLOAD_LENGTH(control);
- u.packet.interrupt = GET_INTERRUPT(control);
- u.packet.skip = GET_SKIP(control);
- u.packet.tag = GET_TAG(control);
- u.packet.sy = GET_SY(control);
- u.packet.header_length = GET_HEADER_LENGTH(control);
+ u->payload_length = GET_PAYLOAD_LENGTH(control);
+ u->interrupt = GET_INTERRUPT(control);
+ u->skip = GET_SKIP(control);
+ u->tag = GET_TAG(control);
+ u->sy = GET_SY(control);
+ u->header_length = GET_HEADER_LENGTH(control);
switch (ctx->type) {
case FW_ISO_CONTEXT_TRANSMIT:
- if (u.packet.header_length & 3)
+ if (u->header_length & 3)
return -EINVAL;
- transmit_header_bytes = u.packet.header_length;
+ transmit_header_bytes = u->header_length;
break;
case FW_ISO_CONTEXT_RECEIVE:
- if (u.packet.header_length == 0 ||
- u.packet.header_length % ctx->header_size != 0)
+ if (u->header_length == 0 ||
+ u->header_length % ctx->header_size != 0)
return -EINVAL;
break;
case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
- if (u.packet.payload_length == 0 ||
- u.packet.payload_length & 3)
+ if (u->payload_length == 0 ||
+ u->payload_length & 3)
return -EINVAL;
break;
}
@@ -1139,20 +1205,19 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)
if (next > end)
return -EINVAL;
if (copy_from_user
- (u.packet.header, p->header, transmit_header_bytes))
+ (u->header, p->header, transmit_header_bytes))
return -EFAULT;
- if (u.packet.skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT &&
- u.packet.header_length + u.packet.payload_length > 0)
+ if (u->skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT &&
+ u->header_length + u->payload_length > 0)
return -EINVAL;
- if (payload + u.packet.payload_length > buffer_end)
+ if (payload + u->payload_length > buffer_end)
return -EINVAL;
- if (fw_iso_context_queue(ctx, &u.packet,
- &client->buffer, payload))
+ if (fw_iso_context_queue(ctx, u, &client->buffer, payload))
break;
p = next;
- payload += u.packet.payload_length;
+ payload += u->payload_length;
count++;
}
fw_iso_context_queue_flush(ctx);
@@ -1212,29 +1277,27 @@ static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg)
struct fw_card *card = client->device->card;
struct timespec64 ts = {0, 0};
u32 cycle_time = 0;
- int ret = 0;
+ int ret;
- local_irq_disable();
+ guard(irq)();
ret = fw_card_read_cycle_time(card, &cycle_time);
if (ret < 0)
- goto end;
+ return ret;
switch (a->clk_id) {
case CLOCK_REALTIME: ktime_get_real_ts64(&ts); break;
case CLOCK_MONOTONIC: ktime_get_ts64(&ts); break;
case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts); break;
default:
- ret = -EINVAL;
+ return -EINVAL;
}
-end:
- local_irq_enable();
a->tv_sec = ts.tv_sec;
a->tv_nsec = ts.tv_nsec;
a->cycle_timer = cycle_time;
- return ret;
+ return 0;
}
static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)
@@ -1254,31 +1317,30 @@ static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)
static void iso_resource_work(struct work_struct *work)
{
struct iso_resource_event *e;
- struct iso_resource *r =
- container_of(work, struct iso_resource, work.work);
+ struct iso_resource *r = from_work(r, work, work.work);
struct client *client = r->client;
+ unsigned long index = r->resource.handle;
int generation, channel, bandwidth, todo;
bool skip, free, success;
- spin_lock_irq(&client->lock);
- generation = client->device->generation;
- todo = r->todo;
- /* Allow 1000ms grace period for other reallocations. */
- if (todo == ISO_RES_ALLOC &&
- time_before64(get_jiffies_64(),
- client->device->card->reset_jiffies + HZ)) {
- schedule_iso_resource(r, DIV_ROUND_UP(HZ, 3));
- skip = true;
- } else {
- /* We could be called twice within the same generation. */
- skip = todo == ISO_RES_REALLOC &&
- r->generation == generation;
+ scoped_guard(spinlock_irq, &client->lock) {
+ generation = client->device->generation;
+ todo = r->todo;
+ // Allow 1000ms grace period for other reallocations.
+ if (todo == ISO_RES_ALLOC &&
+ time_is_after_jiffies64(client->device->card->reset_jiffies + secs_to_jiffies(1))) {
+ schedule_iso_resource(r, msecs_to_jiffies(333));
+ skip = true;
+ } else {
+ // We could be called twice within the same generation.
+ skip = todo == ISO_RES_REALLOC &&
+ r->generation == generation;
+ }
+ free = todo == ISO_RES_DEALLOC ||
+ todo == ISO_RES_ALLOC_ONCE ||
+ todo == ISO_RES_DEALLOC_ONCE;
+ r->generation = generation;
}
- free = todo == ISO_RES_DEALLOC ||
- todo == ISO_RES_ALLOC_ONCE ||
- todo == ISO_RES_DEALLOC_ONCE;
- r->generation = generation;
- spin_unlock_irq(&client->lock);
if (skip)
goto out;
@@ -1292,7 +1354,7 @@ static void iso_resource_work(struct work_struct *work)
todo == ISO_RES_ALLOC_ONCE);
/*
* Is this generation outdated already? As long as this resource sticks
- * in the idr, it will be scheduled again for a newer generation or at
+ * in the xarray, it will be scheduled again for a newer generation or at
* shutdown.
*/
if (channel == -EAGAIN &&
@@ -1301,24 +1363,20 @@ static void iso_resource_work(struct work_struct *work)
success = channel >= 0 || bandwidth > 0;
- spin_lock_irq(&client->lock);
- /*
- * Transit from allocation to reallocation, except if the client
- * requested deallocation in the meantime.
- */
- if (r->todo == ISO_RES_ALLOC)
- r->todo = ISO_RES_REALLOC;
- /*
- * Allocation or reallocation failure? Pull this resource out of the
- * idr and prepare for deletion, unless the client is shutting down.
- */
- if (r->todo == ISO_RES_REALLOC && !success &&
- !client->in_shutdown &&
- idr_remove(&client->resource_idr, r->resource.handle)) {
- client_put(client);
- free = true;
+ scoped_guard(spinlock_irq, &client->lock) {
+ // Transit from allocation to reallocation, except if the client
+ // requested deallocation in the meantime.
+ if (r->todo == ISO_RES_ALLOC)
+ r->todo = ISO_RES_REALLOC;
+ // Allocation or reallocation failure? Pull this resource out of the
+ // xarray and prepare for deletion, unless the client is shutting down.
+ if (r->todo == ISO_RES_REALLOC && !success &&
+ !client->in_shutdown &&
+ xa_erase(&client->resource_xa, index)) {
+ client_put(client);
+ free = true;
+ }
}
- spin_unlock_irq(&client->lock);
if (todo == ISO_RES_ALLOC && channel >= 0)
r->channels = 1ULL << channel;
@@ -1353,13 +1411,12 @@ static void iso_resource_work(struct work_struct *work)
static void release_iso_resource(struct client *client,
struct client_resource *resource)
{
- struct iso_resource *r =
- container_of(resource, struct iso_resource, resource);
+ struct iso_resource *r = to_iso_resource(resource);
+
+ guard(spinlock_irq)(&client->lock);
- spin_lock_irq(&client->lock);
r->todo = ISO_RES_DEALLOC;
schedule_iso_resource(r, 0);
- spin_unlock_irq(&client->lock);
}
static int init_iso_resource(struct client *client,
@@ -1502,26 +1559,64 @@ static void outbound_phy_packet_callback(struct fw_packet *packet,
{
struct outbound_phy_packet_event *e =
container_of(packet, struct outbound_phy_packet_event, p);
- struct client *e_client;
+ struct client *e_client = e->client;
+ u32 rcode;
+
+ trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation,
+ packet->timestamp);
switch (status) {
- /* expected: */
- case ACK_COMPLETE: e->phy_packet.rcode = RCODE_COMPLETE; break;
- /* should never happen with PHY packets: */
- case ACK_PENDING: e->phy_packet.rcode = RCODE_COMPLETE; break;
+ // expected:
+ case ACK_COMPLETE:
+ rcode = RCODE_COMPLETE;
+ break;
+ // should never happen with PHY packets:
+ case ACK_PENDING:
+ rcode = RCODE_COMPLETE;
+ break;
case ACK_BUSY_X:
case ACK_BUSY_A:
- case ACK_BUSY_B: e->phy_packet.rcode = RCODE_BUSY; break;
- case ACK_DATA_ERROR: e->phy_packet.rcode = RCODE_DATA_ERROR; break;
- case ACK_TYPE_ERROR: e->phy_packet.rcode = RCODE_TYPE_ERROR; break;
- /* stale generation; cancelled; on certain controllers: no ack */
- default: e->phy_packet.rcode = status; break;
+ case ACK_BUSY_B:
+ rcode = RCODE_BUSY;
+ break;
+ case ACK_DATA_ERROR:
+ rcode = RCODE_DATA_ERROR;
+ break;
+ case ACK_TYPE_ERROR:
+ rcode = RCODE_TYPE_ERROR;
+ break;
+ // stale generation; cancelled; on certain controllers: no ack
+ default:
+ rcode = status;
+ break;
+ }
+
+ switch (e->phy_packet.without_tstamp.type) {
+ case FW_CDEV_EVENT_PHY_PACKET_SENT:
+ {
+ struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
+
+ pp->rcode = rcode;
+ pp->data[0] = packet->timestamp;
+ queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
+ NULL, 0);
+ break;
+ }
+ case FW_CDEV_EVENT_PHY_PACKET_SENT2:
+ {
+ struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
+
+ pp->rcode = rcode;
+ pp->tstamp = packet->timestamp;
+ queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
+ NULL, 0);
+ break;
+ }
+ default:
+ WARN_ON(1);
+ break;
}
- e->phy_packet.data[0] = packet->timestamp;
- e_client = e->client;
- queue_event(e->client, &e->event, &e->phy_packet,
- sizeof(e->phy_packet) + e->phy_packet.length, NULL, 0);
client_put(e_client);
}
@@ -1535,7 +1630,7 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
if (!client->device->is_local)
return -ENOSYS;
- e = kzalloc(sizeof(*e) + 4, GFP_KERNEL);
+ e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL);
if (e == NULL)
return -ENOMEM;
@@ -1543,15 +1638,32 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
e->client = client;
e->p.speed = SCODE_100;
e->p.generation = a->generation;
- e->p.header[0] = TCODE_LINK_INTERNAL << 4;
+ async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL);
e->p.header[1] = a->data[0];
e->p.header[2] = a->data[1];
e->p.header_length = 12;
e->p.callback = outbound_phy_packet_callback;
- e->phy_packet.closure = a->closure;
- e->phy_packet.type = FW_CDEV_EVENT_PHY_PACKET_SENT;
- if (is_ping_packet(a->data))
- e->phy_packet.length = 4;
+
+ if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
+ struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
+
+ pp->closure = a->closure;
+ pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT;
+ if (is_ping_packet(a->data))
+ pp->length = 4;
+ } else {
+ struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
+
+ pp->closure = a->closure;
+ pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2;
+ // Keep the data field so that application can match the response event to the
+ // request.
+ pp->length = sizeof(a->data);
+ memcpy(pp->data, a->data, sizeof(a->data));
+ }
+
+ trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation,
+ e->p.header[1], e->p.header[2]);
card->driver->send_request(card, &e->p);
@@ -1561,46 +1673,63 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg)
{
struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets;
- struct fw_card *card = client->device->card;
/* Access policy: Allow this ioctl only on local nodes' device files. */
if (!client->device->is_local)
return -ENOSYS;
- spin_lock_irq(&card->lock);
+ // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local
+ // destination never runs in any type of IRQ context.
+ scoped_guard(spinlock_irq, &phy_receiver_list_lock)
+ list_move_tail(&client->phy_receiver_link, &phy_receiver_list);
- list_move_tail(&client->phy_receiver_link, &card->phy_receiver_list);
client->phy_receiver_closure = a->closure;
- spin_unlock_irq(&card->lock);
-
return 0;
}
void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p)
{
struct client *client;
- struct inbound_phy_packet_event *e;
- unsigned long flags;
- spin_lock_irqsave(&card->lock, flags);
+ // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for local
+ // destination never runs in any type of IRQ context.
+ guard(spinlock_irqsave)(&phy_receiver_list_lock);
+
+ list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) {
+ struct inbound_phy_packet_event *e;
+
+ if (client->device->card != card)
+ continue;
- list_for_each_entry(client, &card->phy_receiver_list, phy_receiver_link) {
e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC);
if (e == NULL)
break;
- e->phy_packet.closure = client->phy_receiver_closure;
- e->phy_packet.type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED;
- e->phy_packet.rcode = RCODE_COMPLETE;
- e->phy_packet.length = 8;
- e->phy_packet.data[0] = p->header[1];
- e->phy_packet.data[1] = p->header[2];
- queue_event(client, &e->event,
- &e->phy_packet, sizeof(e->phy_packet) + 8, NULL, 0);
+ if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
+ struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
+
+ pp->closure = client->phy_receiver_closure;
+ pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED;
+ pp->rcode = RCODE_COMPLETE;
+ pp->length = 8;
+ pp->data[0] = p->header[1];
+ pp->data[1] = p->header[2];
+ queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
+ } else {
+ struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
+
+ pp = &e->phy_packet.with_tstamp;
+ pp->closure = client->phy_receiver_closure;
+ pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2;
+ pp->rcode = RCODE_COMPLETE;
+ pp->length = 8;
+ pp->tstamp = p->timestamp;
+ pp->data[0] = p->header[1];
+ pp->data[1] = p->header[2];
+ queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
+ }
}
-
- spin_unlock_irqrestore(&card->lock, flags);
}
static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = {
@@ -1697,16 +1826,15 @@ static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma)
if (ret < 0)
return ret;
- spin_lock_irq(&client->lock);
- if (client->iso_context) {
- ret = fw_iso_buffer_map_dma(&client->buffer,
- client->device->card,
- iso_dma_direction(client->iso_context));
- client->buffer_is_mapped = (ret == 0);
+ scoped_guard(spinlock_irq, &client->lock) {
+ if (client->iso_context) {
+ ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,
+ iso_dma_direction(client->iso_context));
+ if (ret < 0)
+ goto fail;
+ client->buffer_is_mapped = true;
+ }
}
- spin_unlock_irq(&client->lock);
- if (ret < 0)
- goto fail;
ret = vm_map_pages_zero(vma, client->buffer.pages,
client->buffer.page_count);
@@ -1719,48 +1847,35 @@ static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma)
return ret;
}
-static int is_outbound_transaction_resource(int id, void *p, void *data)
+static bool has_outbound_transactions(struct client *client)
{
- struct client_resource *resource = p;
-
- return resource->release == release_transaction;
-}
-
-static int has_outbound_transactions(struct client *client)
-{
- int ret;
-
- spin_lock_irq(&client->lock);
- ret = idr_for_each(&client->resource_idr,
- is_outbound_transaction_resource, NULL);
- spin_unlock_irq(&client->lock);
-
- return ret;
-}
+ struct client_resource *resource;
+ unsigned long index;
-static int shutdown_resource(int id, void *p, void *data)
-{
- struct client_resource *resource = p;
- struct client *client = data;
+ guard(spinlock_irq)(&client->lock);
- resource->release(client, resource);
- client_put(client);
+ xa_for_each(&client->resource_xa, index, resource) {
+ if (is_outbound_transaction_resource(resource))
+ return true;
+ }
- return 0;
+ return false;
}
static int fw_device_op_release(struct inode *inode, struct file *file)
{
struct client *client = file->private_data;
struct event *event, *next_event;
+ struct client_resource *resource;
+ unsigned long index;
- spin_lock_irq(&client->device->card->lock);
- list_del(&client->phy_receiver_link);
- spin_unlock_irq(&client->device->card->lock);
+ // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local
+ // destination never runs in any type of IRQ context.
+ scoped_guard(spinlock_irq, &phy_receiver_list_lock)
+ list_del(&client->phy_receiver_link);
- mutex_lock(&client->device->client_list_mutex);
- list_del(&client->link);
- mutex_unlock(&client->device->client_list_mutex);
+ scoped_guard(mutex, &client->device->client_list_mutex)
+ list_del(&client->link);
if (client->iso_context)
fw_iso_context_destroy(client->iso_context);
@@ -1768,15 +1883,17 @@ static int fw_device_op_release(struct inode *inode, struct file *file)
if (client->buffer.pages)
fw_iso_buffer_destroy(&client->buffer, client->device->card);
- /* Freeze client->resource_idr and client->event_list */
- spin_lock_irq(&client->lock);
- client->in_shutdown = true;
- spin_unlock_irq(&client->lock);
+ // Freeze client->resource_xa and client->event_list.
+ scoped_guard(spinlock_irq, &client->lock)
+ client->in_shutdown = true;
wait_event(client->tx_flush_wait, !has_outbound_transactions(client));
- idr_for_each(&client->resource_idr, shutdown_resource, client);
- idr_destroy(&client->resource_idr);
+ xa_for_each(&client->resource_xa, index, resource) {
+ resource->release(client, resource);
+ client_put(client);
+ }
+ xa_destroy(&client->resource_xa);
list_for_each_entry_safe(event, next_event, &client->event_list, link)
kfree(event);
@@ -1803,7 +1920,6 @@ static __poll_t fw_device_op_poll(struct file *file, poll_table * pt)
const struct file_operations fw_device_ops = {
.owner = THIS_MODULE,
- .llseek = no_llseek,
.open = fw_device_op_open,
.read = fw_device_op_read,
.unlocked_ioctl = fw_device_op_ioctl,