diff options
Diffstat (limited to 'drivers/firewire/core-cdev.c')
| -rw-r--r-- | drivers/firewire/core-cdev.c | 770 |
1 files changed, 443 insertions, 327 deletions
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index 9c89f7d53e99..49dc1612c691 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -14,7 +14,6 @@ #include <linux/errno.h> #include <linux/firewire.h> #include <linux/firewire-cdev.h> -#include <linux/idr.h> #include <linux/irqflags.h> #include <linux/jiffies.h> #include <linux/kernel.h> @@ -35,14 +34,21 @@ #include "core.h" +#include <trace/events/firewire.h> + +#include "packet-header-definitions.h" /* * ABI version history is documented in linux/firewire-cdev.h. */ -#define FW_CDEV_KERNEL_VERSION 5 +#define FW_CDEV_KERNEL_VERSION 6 #define FW_CDEV_VERSION_EVENT_REQUEST2 4 #define FW_CDEV_VERSION_ALLOCATE_REGION_END 4 #define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 5 +#define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP 6 + +static DEFINE_SPINLOCK(phy_receiver_list_lock); +static LIST_HEAD(phy_receiver_list); struct client { u32 version; @@ -50,7 +56,7 @@ struct client { spinlock_t lock; bool in_shutdown; - struct idr resource_idr; + struct xarray resource_xa; struct list_head event_list; wait_queue_head_t wait; wait_queue_head_t tx_flush_wait; @@ -111,6 +117,7 @@ struct inbound_transaction_resource { struct client_resource resource; struct fw_card *card; struct fw_request *request; + bool is_fcp; void *data; size_t length; }; @@ -134,8 +141,41 @@ struct iso_resource { struct iso_resource_event *e_alloc, *e_dealloc; }; +static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource) +{ + return container_of(resource, struct address_handler_resource, resource); +} + +static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource) +{ + return container_of(resource, struct inbound_transaction_resource, resource); +} + +static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource) +{ + return container_of(resource, struct descriptor_resource, resource); +} + +static struct iso_resource *to_iso_resource(struct client_resource *resource) +{ + return container_of(resource, struct iso_resource, resource); +} + static void release_iso_resource(struct client *, struct client_resource *); +static int is_iso_resource(const struct client_resource *resource) +{ + return resource->release == release_iso_resource; +} + +static void release_transaction(struct client *client, + struct client_resource *resource); + +static int is_outbound_transaction_resource(const struct client_resource *resource) +{ + return resource->release == release_transaction; +} + static void schedule_iso_resource(struct iso_resource *r, unsigned long delay) { client_get(r->client); @@ -143,13 +183,6 @@ static void schedule_iso_resource(struct iso_resource *r, unsigned long delay) client_put(r->client); } -static void schedule_if_iso_resource(struct client_resource *resource) -{ - if (resource->release == release_iso_resource) - schedule_iso_resource(container_of(resource, - struct iso_resource, resource), 0); -} - /* * dequeue_event() just kfree()'s the event, so the event has to be * the first field in a struct XYZ_event. @@ -168,7 +201,10 @@ struct outbound_transaction_event { struct event event; struct client *client; struct outbound_transaction_resource r; - struct fw_cdev_event_response response; + union { + struct fw_cdev_event_response without_tstamp; + struct fw_cdev_event_response2 with_tstamp; + } rsp; }; struct inbound_transaction_event { @@ -176,6 +212,7 @@ struct inbound_transaction_event { union { struct fw_cdev_event_request request; struct fw_cdev_event_request2 request2; + struct fw_cdev_event_request3 with_tstamp; } req; }; @@ -198,12 +235,18 @@ struct outbound_phy_packet_event { struct event event; struct client *client; struct fw_packet p; - struct fw_cdev_event_phy_packet phy_packet; + union { + struct fw_cdev_event_phy_packet without_tstamp; + struct fw_cdev_event_phy_packet2 with_tstamp; + } phy_packet; }; struct inbound_phy_packet_event { struct event event; - struct fw_cdev_event_phy_packet phy_packet; + union { + struct fw_cdev_event_phy_packet without_tstamp; + struct fw_cdev_event_phy_packet2 with_tstamp; + } phy_packet; }; #ifdef CONFIG_COMPAT @@ -256,7 +299,7 @@ static int fw_device_op_open(struct inode *inode, struct file *file) client->device = device; spin_lock_init(&client->lock); - idr_init(&client->resource_idr); + xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH); INIT_LIST_HEAD(&client->event_list); init_waitqueue_head(&client->wait); init_waitqueue_head(&client->tx_flush_wait); @@ -272,19 +315,17 @@ static int fw_device_op_open(struct inode *inode, struct file *file) static void queue_event(struct client *client, struct event *event, void *data0, size_t size0, void *data1, size_t size1) { - unsigned long flags; - event->v[0].data = data0; event->v[0].size = size0; event->v[1].data = data1; event->v[1].size = size1; - spin_lock_irqsave(&client->lock, flags); - if (client->in_shutdown) - kfree(event); - else - list_add_tail(&event->link, &client->event_list); - spin_unlock_irqrestore(&client->lock, flags); + scoped_guard(spinlock_irqsave, &client->lock) { + if (client->in_shutdown) + kfree(event); + else + list_add_tail(&event->link, &client->event_list); + } wake_up_interruptible(&client->wait); } @@ -306,10 +347,10 @@ static int dequeue_event(struct client *client, fw_device_is_shutdown(client->device)) return -ENODEV; - spin_lock_irq(&client->lock); - event = list_first_entry(&client->event_list, struct event, link); - list_del(&event->link); - spin_unlock_irq(&client->lock); + scoped_guard(spinlock_irq, &client->lock) { + event = list_first_entry(&client->event_list, struct event, link); + list_del(&event->link); + } total = 0; for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) { @@ -341,7 +382,7 @@ static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event, { struct fw_card *card = client->device->card; - spin_lock_irq(&card->lock); + guard(spinlock_irq)(&card->lock); event->closure = client->bus_reset_closure; event->type = FW_CDEV_EVENT_BUS_RESET; @@ -351,8 +392,6 @@ static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event, event->bm_node_id = card->bm_node_id; event->irm_node_id = card->irm_node->node_id; event->root_node_id = card->root_node->node_id; - - spin_unlock_irq(&card->lock); } static void for_each_client(struct fw_device *device, @@ -360,22 +399,17 @@ static void for_each_client(struct fw_device *device, { struct client *c; - mutex_lock(&device->client_list_mutex); + guard(mutex)(&device->client_list_mutex); + list_for_each_entry(c, &device->client_list, link) callback(c); - mutex_unlock(&device->client_list_mutex); -} - -static int schedule_reallocations(int id, void *p, void *data) -{ - schedule_if_iso_resource(p); - - return 0; } static void queue_bus_reset_event(struct client *client) { struct bus_reset_event *e; + struct client_resource *resource; + unsigned long index; e = kzalloc(sizeof(*e), GFP_KERNEL); if (e == NULL) @@ -386,9 +420,12 @@ static void queue_bus_reset_event(struct client *client) queue_event(client, &e->event, &e->reset, sizeof(e->reset), NULL, 0); - spin_lock_irq(&client->lock); - idr_for_each(&client->resource_idr, schedule_reallocations, client); - spin_unlock_irq(&client->lock); + guard(spinlock_irq)(&client->lock); + + xa_for_each(&client->resource_xa, index, resource) { + if (is_iso_resource(resource)) + schedule_iso_resource(to_iso_resource(resource), 0); + } } void fw_device_cdev_update(struct fw_device *device) @@ -439,23 +476,20 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg) a->version = FW_CDEV_KERNEL_VERSION; a->card = client->device->card->index; - down_read(&fw_device_rwsem); + scoped_guard(rwsem_read, &fw_device_rwsem) { + if (a->rom != 0) { + size_t want = a->rom_length; + size_t have = client->device->config_rom_length * 4; - if (a->rom != 0) { - size_t want = a->rom_length; - size_t have = client->device->config_rom_length * 4; - - ret = copy_to_user(u64_to_uptr(a->rom), - client->device->config_rom, min(want, have)); + ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom, + min(want, have)); + if (ret != 0) + return -EFAULT; + } + a->rom_length = client->device->config_rom_length * 4; } - a->rom_length = client->device->config_rom_length * 4; - - up_read(&fw_device_rwsem); - if (ret != 0) - return -EFAULT; - - mutex_lock(&client->device->client_list_mutex); + guard(mutex)(&client->device->client_list_mutex); client->bus_reset_closure = a->bus_reset_closure; if (a->bus_reset != 0) { @@ -466,37 +500,36 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg) if (ret == 0 && list_empty(&client->link)) list_add_tail(&client->link, &client->device->client_list); - mutex_unlock(&client->device->client_list_mutex); - return ret ? -EFAULT : 0; } -static int add_client_resource(struct client *client, - struct client_resource *resource, gfp_t gfp_mask) +static int add_client_resource(struct client *client, struct client_resource *resource, + gfp_t gfp_mask) { - bool preload = gfpflags_allow_blocking(gfp_mask); - unsigned long flags; int ret; - if (preload) - idr_preload(gfp_mask); - spin_lock_irqsave(&client->lock, flags); - - if (client->in_shutdown) - ret = -ECANCELED; - else - ret = idr_alloc(&client->resource_idr, resource, 0, 0, - GFP_NOWAIT); - if (ret >= 0) { - resource->handle = ret; - client_get(client); - schedule_if_iso_resource(resource); + scoped_guard(spinlock_irqsave, &client->lock) { + u32 index; + + if (client->in_shutdown) { + ret = -ECANCELED; + } else { + if (gfpflags_allow_blocking(gfp_mask)) { + ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b, + GFP_NOWAIT); + } else { + ret = xa_alloc_bh(&client->resource_xa, &index, resource, + xa_limit_32b, GFP_NOWAIT); + } + } + if (ret >= 0) { + resource->handle = index; + client_get(client); + if (is_iso_resource(resource)) + schedule_iso_resource(to_iso_resource(resource), 0); + } } - spin_unlock_irqrestore(&client->lock, flags); - if (preload) - idr_preload_end(); - return ret < 0 ? ret : 0; } @@ -504,19 +537,19 @@ static int release_client_resource(struct client *client, u32 handle, client_resource_release_fn_t release, struct client_resource **return_resource) { + unsigned long index = handle; struct client_resource *resource; - spin_lock_irq(&client->lock); - if (client->in_shutdown) - resource = NULL; - else - resource = idr_find(&client->resource_idr, handle); - if (resource && resource->release == release) - idr_remove(&client->resource_idr, handle); - spin_unlock_irq(&client->lock); + scoped_guard(spinlock_irq, &client->lock) { + if (client->in_shutdown) + return -EINVAL; - if (!(resource && resource->release == release)) - return -EINVAL; + resource = xa_load(&client->resource_xa, index); + if (!resource || resource->release != release) + return -EINVAL; + + xa_erase(&client->resource_xa, handle); + } if (return_resource) *return_resource = resource; @@ -533,43 +566,66 @@ static void release_transaction(struct client *client, { } -static void complete_transaction(struct fw_card *card, int rcode, - void *payload, size_t length, void *data) +static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp, + u32 response_tstamp, void *payload, size_t length, void *data) { struct outbound_transaction_event *e = data; - struct fw_cdev_event_response *rsp = &e->response; struct client *client = e->client; - unsigned long flags; + unsigned long index = e->r.resource.handle; - if (length < rsp->length) - rsp->length = length; - if (rcode == RCODE_COMPLETE) - memcpy(rsp->data, payload, rsp->length); + scoped_guard(spinlock_irqsave, &client->lock) { + xa_erase(&client->resource_xa, index); + if (client->in_shutdown) + wake_up(&client->tx_flush_wait); + } - spin_lock_irqsave(&client->lock, flags); - idr_remove(&client->resource_idr, e->r.resource.handle); - if (client->in_shutdown) - wake_up(&client->tx_flush_wait); - spin_unlock_irqrestore(&client->lock, flags); + switch (e->rsp.without_tstamp.type) { + case FW_CDEV_EVENT_RESPONSE: + { + struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp; + + if (length < rsp->length) + rsp->length = length; + if (rcode == RCODE_COMPLETE) + memcpy(rsp->data, payload, rsp->length); + + rsp->rcode = rcode; + + // In the case that sizeof(*rsp) doesn't align with the position of the + // data, and the read is short, preserve an extra copy of the data + // to stay compatible with a pre-2.6.27 bug. Since the bug is harmless + // for short reads and some apps depended on it, this is both safe + // and prudent for compatibility. + if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data)) + queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length); + else + queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0); - rsp->type = FW_CDEV_EVENT_RESPONSE; - rsp->rcode = rcode; + break; + } + case FW_CDEV_EVENT_RESPONSE2: + { + struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp; - /* - * In the case that sizeof(*rsp) doesn't align with the position of the - * data, and the read is short, preserve an extra copy of the data - * to stay compatible with a pre-2.6.27 bug. Since the bug is harmless - * for short reads and some apps depended on it, this is both safe - * and prudent for compatibility. - */ - if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data)) - queue_event(client, &e->event, rsp, sizeof(*rsp), - rsp->data, rsp->length); - else - queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, - NULL, 0); + if (length < rsp->length) + rsp->length = length; + if (rcode == RCODE_COMPLETE) + memcpy(rsp->data, payload, rsp->length); + + rsp->rcode = rcode; + rsp->request_tstamp = request_tstamp; + rsp->response_tstamp = response_tstamp; - /* Drop the idr's reference */ + queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0); + + break; + } + default: + WARN_ON(1); + break; + } + + // Drop the xarray's reference. client_put(client); } @@ -578,6 +634,7 @@ static int init_request(struct client *client, int destination_id, int speed) { struct outbound_transaction_event *e; + void *payload; int ret; if (request->tcode != TCODE_STREAM_DATA && @@ -591,14 +648,25 @@ static int init_request(struct client *client, e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL); if (e == NULL) return -ENOMEM; - e->client = client; - e->response.length = request->length; - e->response.closure = request->closure; - if (request->data && - copy_from_user(e->response.data, - u64_to_uptr(request->data), request->length)) { + if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { + struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp; + + rsp->type = FW_CDEV_EVENT_RESPONSE; + rsp->length = request->length; + rsp->closure = request->closure; + payload = rsp->data; + } else { + struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp; + + rsp->type = FW_CDEV_EVENT_RESPONSE2; + rsp->length = request->length; + rsp->closure = request->closure; + payload = rsp->data; + } + + if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) { ret = -EFAULT; goto failed; } @@ -608,10 +676,9 @@ static int init_request(struct client *client, if (ret < 0) goto failed; - fw_send_request(client->device->card, &e->r.transaction, - request->tcode, destination_id, request->generation, - speed, request->offset, e->response.data, - request->length, complete_transaction, e); + fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode, + destination_id, request->generation, speed, request->offset, + payload, request->length, complete_transaction, e); return 0; failed: @@ -643,19 +710,13 @@ static int ioctl_send_request(struct client *client, union ioctl_arg *arg) client->device->max_speed); } -static inline bool is_fcp_request(struct fw_request *request) -{ - return request == NULL; -} - static void release_request(struct client *client, struct client_resource *resource) { - struct inbound_transaction_resource *r = container_of(resource, - struct inbound_transaction_resource, resource); + struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource); - if (is_fcp_request(r->request)) - kfree(r->data); + if (r->is_fcp) + fw_request_put(r->request); else fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR); @@ -669,15 +730,20 @@ static void handle_request(struct fw_card *card, struct fw_request *request, void *payload, size_t length, void *callback_data) { struct address_handler_resource *handler = callback_data; + bool is_fcp = is_in_fcp_region(offset, length); struct inbound_transaction_resource *r; struct inbound_transaction_event *e; size_t event_size0; - void *fcp_frame = NULL; int ret; /* card may be different from handler->client->device->card */ fw_card_get(card); + // Extend the lifetime of data for request so that its payload is safely accessible in + // the process context for the client. + if (is_fcp) + fw_request_get(request); + r = kmalloc(sizeof(*r), GFP_ATOMIC); e = kmalloc(sizeof(*e), GFP_ATOMIC); if (r == NULL || e == NULL) @@ -685,21 +751,10 @@ static void handle_request(struct fw_card *card, struct fw_request *request, r->card = card; r->request = request; + r->is_fcp = is_fcp; r->data = payload; r->length = length; - if (is_fcp_request(request)) { - /* - * FIXME: Let core-transaction.c manage a - * single reference-counted copy? - */ - fcp_frame = kmemdup(payload, length, GFP_ATOMIC); - if (fcp_frame == NULL) - goto failed; - - r->data = fcp_frame; - } - r->resource.release = release_request; ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC); if (ret < 0) @@ -718,7 +773,7 @@ static void handle_request(struct fw_card *card, struct fw_request *request, req->handle = r->resource.handle; req->closure = handler->closure; event_size0 = sizeof(*req); - } else { + } else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { struct fw_cdev_event_request2 *req = &e->req.request2; req->type = FW_CDEV_EVENT_REQUEST2; @@ -732,6 +787,21 @@ static void handle_request(struct fw_card *card, struct fw_request *request, req->handle = r->resource.handle; req->closure = handler->closure; event_size0 = sizeof(*req); + } else { + struct fw_cdev_event_request3 *req = &e->req.with_tstamp; + + req->type = FW_CDEV_EVENT_REQUEST3; + req->tcode = tcode; + req->offset = offset; + req->source_node_id = source; + req->destination_node_id = destination; + req->card = card->index; + req->generation = generation; + req->length = length; + req->handle = r->resource.handle; + req->closure = handler->closure; + req->tstamp = fw_request_get_timestamp(request); + event_size0 = sizeof(*req); } queue_event(handler->client, &e->event, @@ -741,10 +811,11 @@ static void handle_request(struct fw_card *card, struct fw_request *request, failed: kfree(r); kfree(e); - kfree(fcp_frame); - if (!is_fcp_request(request)) + if (!is_fcp) fw_send_response(card, request, RCODE_CONFLICT_ERROR); + else + fw_request_put(request); fw_card_put(card); } @@ -752,8 +823,7 @@ static void handle_request(struct fw_card *card, struct fw_request *request, static void release_address_handler(struct client *client, struct client_resource *resource) { - struct address_handler_resource *r = - container_of(resource, struct address_handler_resource, resource); + struct address_handler_resource *r = to_address_handler_resource(resource); fw_core_remove_address_handler(&r->handler); kfree(r); @@ -817,19 +887,20 @@ static int ioctl_send_response(struct client *client, union ioctl_arg *arg) release_request, &resource) < 0) return -EINVAL; - r = container_of(resource, struct inbound_transaction_resource, - resource); - if (is_fcp_request(r->request)) + r = to_inbound_transaction_resource(resource); + if (r->is_fcp) { + fw_request_put(r->request); goto out; + } if (a->length != fw_get_response_length(r->request)) { ret = -EINVAL; - kfree(r->request); + fw_request_put(r->request); goto out; } if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) { ret = -EFAULT; - kfree(r->request); + fw_request_put(r->request); goto out; } fw_send_response(r->card, r->request, a->rcode); @@ -850,8 +921,7 @@ static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg) static void release_descriptor(struct client *client, struct client_resource *resource) { - struct descriptor_resource *r = - container_of(resource, struct descriptor_resource, resource); + struct descriptor_resource *r = to_descriptor_resource(resource); fw_core_remove_descriptor(&r->descriptor); kfree(r); @@ -870,11 +940,12 @@ static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg) if (a->length > 256) return -EINVAL; - r = kmalloc(sizeof(*r) + a->length * 4, GFP_KERNEL); + r = kmalloc(struct_size(r, data, a->length), GFP_KERNEL); if (r == NULL) return -ENOMEM; - if (copy_from_user(r->data, u64_to_uptr(a->data), a->length * 4)) { + if (copy_from_user(r->data, u64_to_uptr(a->data), + flex_array_size(r, data, a->length))) { ret = -EFAULT; goto failed; } @@ -915,7 +986,7 @@ static void iso_callback(struct fw_iso_context *context, u32 cycle, struct client *client = data; struct iso_interrupt_event *e; - e = kmalloc(sizeof(*e) + header_length, GFP_ATOMIC); + e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL); if (e == NULL) return; @@ -934,7 +1005,7 @@ static void iso_mc_callback(struct fw_iso_context *context, struct client *client = data; struct iso_interrupt_mc_event *e; - e = kmalloc(sizeof(*e), GFP_ATOMIC); + e = kmalloc(sizeof(*e), GFP_KERNEL); if (e == NULL) return; @@ -1016,10 +1087,10 @@ static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg) if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW) context->drop_overflow_headers = true; - /* We only support one context at this time. */ - spin_lock_irq(&client->lock); + // We only support one context at this time. + guard(spinlock_irq)(&client->lock); + if (client->iso_context != NULL) { - spin_unlock_irq(&client->lock); fw_iso_context_destroy(context); return -EBUSY; @@ -1029,7 +1100,6 @@ static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg) client->device->card, iso_dma_direction(context)); if (ret < 0) { - spin_unlock_irq(&client->lock); fw_iso_context_destroy(context); return ret; @@ -1038,7 +1108,6 @@ static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg) } client->iso_closure = a->closure; client->iso_context = context; - spin_unlock_irq(&client->lock); a->handle = 0; @@ -1072,10 +1141,7 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg) unsigned long payload, buffer_end, transmit_header_bytes = 0; u32 control; int count; - struct { - struct fw_iso_packet packet; - u8 header[256]; - } u; + DEFINE_RAW_FLEX(struct fw_iso_packet, u, header, 64); if (ctx == NULL || a->handle != 0) return -EINVAL; @@ -1107,29 +1173,29 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg) while (p < end) { if (get_user(control, &p->control)) return -EFAULT; - u.packet.payload_length = GET_PAYLOAD_LENGTH(control); - u.packet.interrupt = GET_INTERRUPT(control); - u.packet.skip = GET_SKIP(control); - u.packet.tag = GET_TAG(control); - u.packet.sy = GET_SY(control); - u.packet.header_length = GET_HEADER_LENGTH(control); + u->payload_length = GET_PAYLOAD_LENGTH(control); + u->interrupt = GET_INTERRUPT(control); + u->skip = GET_SKIP(control); + u->tag = GET_TAG(control); + u->sy = GET_SY(control); + u->header_length = GET_HEADER_LENGTH(control); switch (ctx->type) { case FW_ISO_CONTEXT_TRANSMIT: - if (u.packet.header_length & 3) + if (u->header_length & 3) return -EINVAL; - transmit_header_bytes = u.packet.header_length; + transmit_header_bytes = u->header_length; break; case FW_ISO_CONTEXT_RECEIVE: - if (u.packet.header_length == 0 || - u.packet.header_length % ctx->header_size != 0) + if (u->header_length == 0 || + u->header_length % ctx->header_size != 0) return -EINVAL; break; case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL: - if (u.packet.payload_length == 0 || - u.packet.payload_length & 3) + if (u->payload_length == 0 || + u->payload_length & 3) return -EINVAL; break; } @@ -1139,20 +1205,19 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg) if (next > end) return -EINVAL; if (copy_from_user - (u.packet.header, p->header, transmit_header_bytes)) + (u->header, p->header, transmit_header_bytes)) return -EFAULT; - if (u.packet.skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT && - u.packet.header_length + u.packet.payload_length > 0) + if (u->skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT && + u->header_length + u->payload_length > 0) return -EINVAL; - if (payload + u.packet.payload_length > buffer_end) + if (payload + u->payload_length > buffer_end) return -EINVAL; - if (fw_iso_context_queue(ctx, &u.packet, - &client->buffer, payload)) + if (fw_iso_context_queue(ctx, u, &client->buffer, payload)) break; p = next; - payload += u.packet.payload_length; + payload += u->payload_length; count++; } fw_iso_context_queue_flush(ctx); @@ -1212,29 +1277,27 @@ static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg) struct fw_card *card = client->device->card; struct timespec64 ts = {0, 0}; u32 cycle_time = 0; - int ret = 0; + int ret; - local_irq_disable(); + guard(irq)(); ret = fw_card_read_cycle_time(card, &cycle_time); if (ret < 0) - goto end; + return ret; switch (a->clk_id) { case CLOCK_REALTIME: ktime_get_real_ts64(&ts); break; case CLOCK_MONOTONIC: ktime_get_ts64(&ts); break; case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts); break; default: - ret = -EINVAL; + return -EINVAL; } -end: - local_irq_enable(); a->tv_sec = ts.tv_sec; a->tv_nsec = ts.tv_nsec; a->cycle_timer = cycle_time; - return ret; + return 0; } static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg) @@ -1254,31 +1317,30 @@ static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg) static void iso_resource_work(struct work_struct *work) { struct iso_resource_event *e; - struct iso_resource *r = - container_of(work, struct iso_resource, work.work); + struct iso_resource *r = from_work(r, work, work.work); struct client *client = r->client; + unsigned long index = r->resource.handle; int generation, channel, bandwidth, todo; bool skip, free, success; - spin_lock_irq(&client->lock); - generation = client->device->generation; - todo = r->todo; - /* Allow 1000ms grace period for other reallocations. */ - if (todo == ISO_RES_ALLOC && - time_before64(get_jiffies_64(), - client->device->card->reset_jiffies + HZ)) { - schedule_iso_resource(r, DIV_ROUND_UP(HZ, 3)); - skip = true; - } else { - /* We could be called twice within the same generation. */ - skip = todo == ISO_RES_REALLOC && - r->generation == generation; + scoped_guard(spinlock_irq, &client->lock) { + generation = client->device->generation; + todo = r->todo; + // Allow 1000ms grace period for other reallocations. + if (todo == ISO_RES_ALLOC && + time_is_after_jiffies64(client->device->card->reset_jiffies + secs_to_jiffies(1))) { + schedule_iso_resource(r, msecs_to_jiffies(333)); + skip = true; + } else { + // We could be called twice within the same generation. + skip = todo == ISO_RES_REALLOC && + r->generation == generation; + } + free = todo == ISO_RES_DEALLOC || + todo == ISO_RES_ALLOC_ONCE || + todo == ISO_RES_DEALLOC_ONCE; + r->generation = generation; } - free = todo == ISO_RES_DEALLOC || - todo == ISO_RES_ALLOC_ONCE || - todo == ISO_RES_DEALLOC_ONCE; - r->generation = generation; - spin_unlock_irq(&client->lock); if (skip) goto out; @@ -1292,7 +1354,7 @@ static void iso_resource_work(struct work_struct *work) todo == ISO_RES_ALLOC_ONCE); /* * Is this generation outdated already? As long as this resource sticks - * in the idr, it will be scheduled again for a newer generation or at + * in the xarray, it will be scheduled again for a newer generation or at * shutdown. */ if (channel == -EAGAIN && @@ -1301,24 +1363,20 @@ static void iso_resource_work(struct work_struct *work) success = channel >= 0 || bandwidth > 0; - spin_lock_irq(&client->lock); - /* - * Transit from allocation to reallocation, except if the client - * requested deallocation in the meantime. - */ - if (r->todo == ISO_RES_ALLOC) - r->todo = ISO_RES_REALLOC; - /* - * Allocation or reallocation failure? Pull this resource out of the - * idr and prepare for deletion, unless the client is shutting down. - */ - if (r->todo == ISO_RES_REALLOC && !success && - !client->in_shutdown && - idr_remove(&client->resource_idr, r->resource.handle)) { - client_put(client); - free = true; + scoped_guard(spinlock_irq, &client->lock) { + // Transit from allocation to reallocation, except if the client + // requested deallocation in the meantime. + if (r->todo == ISO_RES_ALLOC) + r->todo = ISO_RES_REALLOC; + // Allocation or reallocation failure? Pull this resource out of the + // xarray and prepare for deletion, unless the client is shutting down. + if (r->todo == ISO_RES_REALLOC && !success && + !client->in_shutdown && + xa_erase(&client->resource_xa, index)) { + client_put(client); + free = true; + } } - spin_unlock_irq(&client->lock); if (todo == ISO_RES_ALLOC && channel >= 0) r->channels = 1ULL << channel; @@ -1353,13 +1411,12 @@ static void iso_resource_work(struct work_struct *work) static void release_iso_resource(struct client *client, struct client_resource *resource) { - struct iso_resource *r = - container_of(resource, struct iso_resource, resource); + struct iso_resource *r = to_iso_resource(resource); + + guard(spinlock_irq)(&client->lock); - spin_lock_irq(&client->lock); r->todo = ISO_RES_DEALLOC; schedule_iso_resource(r, 0); - spin_unlock_irq(&client->lock); } static int init_iso_resource(struct client *client, @@ -1502,26 +1559,64 @@ static void outbound_phy_packet_callback(struct fw_packet *packet, { struct outbound_phy_packet_event *e = container_of(packet, struct outbound_phy_packet_event, p); - struct client *e_client; + struct client *e_client = e->client; + u32 rcode; + + trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation, + packet->timestamp); switch (status) { - /* expected: */ - case ACK_COMPLETE: e->phy_packet.rcode = RCODE_COMPLETE; break; - /* should never happen with PHY packets: */ - case ACK_PENDING: e->phy_packet.rcode = RCODE_COMPLETE; break; + // expected: + case ACK_COMPLETE: + rcode = RCODE_COMPLETE; + break; + // should never happen with PHY packets: + case ACK_PENDING: + rcode = RCODE_COMPLETE; + break; case ACK_BUSY_X: case ACK_BUSY_A: - case ACK_BUSY_B: e->phy_packet.rcode = RCODE_BUSY; break; - case ACK_DATA_ERROR: e->phy_packet.rcode = RCODE_DATA_ERROR; break; - case ACK_TYPE_ERROR: e->phy_packet.rcode = RCODE_TYPE_ERROR; break; - /* stale generation; cancelled; on certain controllers: no ack */ - default: e->phy_packet.rcode = status; break; + case ACK_BUSY_B: + rcode = RCODE_BUSY; + break; + case ACK_DATA_ERROR: + rcode = RCODE_DATA_ERROR; + break; + case ACK_TYPE_ERROR: + rcode = RCODE_TYPE_ERROR; + break; + // stale generation; cancelled; on certain controllers: no ack + default: + rcode = status; + break; + } + + switch (e->phy_packet.without_tstamp.type) { + case FW_CDEV_EVENT_PHY_PACKET_SENT: + { + struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp; + + pp->rcode = rcode; + pp->data[0] = packet->timestamp; + queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length, + NULL, 0); + break; + } + case FW_CDEV_EVENT_PHY_PACKET_SENT2: + { + struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp; + + pp->rcode = rcode; + pp->tstamp = packet->timestamp; + queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length, + NULL, 0); + break; + } + default: + WARN_ON(1); + break; } - e->phy_packet.data[0] = packet->timestamp; - e_client = e->client; - queue_event(e->client, &e->event, &e->phy_packet, - sizeof(e->phy_packet) + e->phy_packet.length, NULL, 0); client_put(e_client); } @@ -1535,7 +1630,7 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg) if (!client->device->is_local) return -ENOSYS; - e = kzalloc(sizeof(*e) + 4, GFP_KERNEL); + e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL); if (e == NULL) return -ENOMEM; @@ -1543,15 +1638,32 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg) e->client = client; e->p.speed = SCODE_100; e->p.generation = a->generation; - e->p.header[0] = TCODE_LINK_INTERNAL << 4; + async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL); e->p.header[1] = a->data[0]; e->p.header[2] = a->data[1]; e->p.header_length = 12; e->p.callback = outbound_phy_packet_callback; - e->phy_packet.closure = a->closure; - e->phy_packet.type = FW_CDEV_EVENT_PHY_PACKET_SENT; - if (is_ping_packet(a->data)) - e->phy_packet.length = 4; + + if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { + struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp; + + pp->closure = a->closure; + pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT; + if (is_ping_packet(a->data)) + pp->length = 4; + } else { + struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp; + + pp->closure = a->closure; + pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2; + // Keep the data field so that application can match the response event to the + // request. + pp->length = sizeof(a->data); + memcpy(pp->data, a->data, sizeof(a->data)); + } + + trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation, + e->p.header[1], e->p.header[2]); card->driver->send_request(card, &e->p); @@ -1561,46 +1673,63 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg) static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg) { struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets; - struct fw_card *card = client->device->card; /* Access policy: Allow this ioctl only on local nodes' device files. */ if (!client->device->is_local) return -ENOSYS; - spin_lock_irq(&card->lock); + // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local + // destination never runs in any type of IRQ context. + scoped_guard(spinlock_irq, &phy_receiver_list_lock) + list_move_tail(&client->phy_receiver_link, &phy_receiver_list); - list_move_tail(&client->phy_receiver_link, &card->phy_receiver_list); client->phy_receiver_closure = a->closure; - spin_unlock_irq(&card->lock); - return 0; } void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p) { struct client *client; - struct inbound_phy_packet_event *e; - unsigned long flags; - spin_lock_irqsave(&card->lock, flags); + // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for local + // destination never runs in any type of IRQ context. + guard(spinlock_irqsave)(&phy_receiver_list_lock); + + list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) { + struct inbound_phy_packet_event *e; + + if (client->device->card != card) + continue; - list_for_each_entry(client, &card->phy_receiver_list, phy_receiver_link) { e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC); if (e == NULL) break; - e->phy_packet.closure = client->phy_receiver_closure; - e->phy_packet.type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED; - e->phy_packet.rcode = RCODE_COMPLETE; - e->phy_packet.length = 8; - e->phy_packet.data[0] = p->header[1]; - e->phy_packet.data[1] = p->header[2]; - queue_event(client, &e->event, - &e->phy_packet, sizeof(e->phy_packet) + 8, NULL, 0); + if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { + struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp; + + pp->closure = client->phy_receiver_closure; + pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED; + pp->rcode = RCODE_COMPLETE; + pp->length = 8; + pp->data[0] = p->header[1]; + pp->data[1] = p->header[2]; + queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0); + } else { + struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp; + + pp = &e->phy_packet.with_tstamp; + pp->closure = client->phy_receiver_closure; + pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2; + pp->rcode = RCODE_COMPLETE; + pp->length = 8; + pp->tstamp = p->timestamp; + pp->data[0] = p->header[1]; + pp->data[1] = p->header[2]; + queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0); + } } - - spin_unlock_irqrestore(&card->lock, flags); } static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = { @@ -1697,16 +1826,15 @@ static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma) if (ret < 0) return ret; - spin_lock_irq(&client->lock); - if (client->iso_context) { - ret = fw_iso_buffer_map_dma(&client->buffer, - client->device->card, - iso_dma_direction(client->iso_context)); - client->buffer_is_mapped = (ret == 0); + scoped_guard(spinlock_irq, &client->lock) { + if (client->iso_context) { + ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card, + iso_dma_direction(client->iso_context)); + if (ret < 0) + goto fail; + client->buffer_is_mapped = true; + } } - spin_unlock_irq(&client->lock); - if (ret < 0) - goto fail; ret = vm_map_pages_zero(vma, client->buffer.pages, client->buffer.page_count); @@ -1719,48 +1847,35 @@ static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma) return ret; } -static int is_outbound_transaction_resource(int id, void *p, void *data) +static bool has_outbound_transactions(struct client *client) { - struct client_resource *resource = p; - - return resource->release == release_transaction; -} - -static int has_outbound_transactions(struct client *client) -{ - int ret; - - spin_lock_irq(&client->lock); - ret = idr_for_each(&client->resource_idr, - is_outbound_transaction_resource, NULL); - spin_unlock_irq(&client->lock); - - return ret; -} + struct client_resource *resource; + unsigned long index; -static int shutdown_resource(int id, void *p, void *data) -{ - struct client_resource *resource = p; - struct client *client = data; + guard(spinlock_irq)(&client->lock); - resource->release(client, resource); - client_put(client); + xa_for_each(&client->resource_xa, index, resource) { + if (is_outbound_transaction_resource(resource)) + return true; + } - return 0; + return false; } static int fw_device_op_release(struct inode *inode, struct file *file) { struct client *client = file->private_data; struct event *event, *next_event; + struct client_resource *resource; + unsigned long index; - spin_lock_irq(&client->device->card->lock); - list_del(&client->phy_receiver_link); - spin_unlock_irq(&client->device->card->lock); + // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local + // destination never runs in any type of IRQ context. + scoped_guard(spinlock_irq, &phy_receiver_list_lock) + list_del(&client->phy_receiver_link); - mutex_lock(&client->device->client_list_mutex); - list_del(&client->link); - mutex_unlock(&client->device->client_list_mutex); + scoped_guard(mutex, &client->device->client_list_mutex) + list_del(&client->link); if (client->iso_context) fw_iso_context_destroy(client->iso_context); @@ -1768,15 +1883,17 @@ static int fw_device_op_release(struct inode *inode, struct file *file) if (client->buffer.pages) fw_iso_buffer_destroy(&client->buffer, client->device->card); - /* Freeze client->resource_idr and client->event_list */ - spin_lock_irq(&client->lock); - client->in_shutdown = true; - spin_unlock_irq(&client->lock); + // Freeze client->resource_xa and client->event_list. + scoped_guard(spinlock_irq, &client->lock) + client->in_shutdown = true; wait_event(client->tx_flush_wait, !has_outbound_transactions(client)); - idr_for_each(&client->resource_idr, shutdown_resource, client); - idr_destroy(&client->resource_idr); + xa_for_each(&client->resource_xa, index, resource) { + resource->release(client, resource); + client_put(client); + } + xa_destroy(&client->resource_xa); list_for_each_entry_safe(event, next_event, &client->event_list, link) kfree(event); @@ -1803,7 +1920,6 @@ static __poll_t fw_device_op_poll(struct file *file, poll_table * pt) const struct file_operations fw_device_ops = { .owner = THIS_MODULE, - .llseek = no_llseek, .open = fw_device_op_open, .read = fw_device_op_read, .unlocked_ioctl = fw_device_op_ioctl, |
