diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h')
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h | 160 |
1 files changed, 138 insertions, 22 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index 71e2fde56b2b..74a61b5b2f0b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -1,5 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /* - * Copyright 2014 Advanced Micro Devices, Inc. + * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -34,11 +35,49 @@ #define VMID_NUM 16 +#define KFD_MES_PROCESS_QUANTUM 100000 +#define KFD_MES_GANG_QUANTUM 10000 + struct device_process_node { struct qcm_process_device *qpd; struct list_head list; }; +union SQ_CMD_BITS { + struct { + uint32_t cmd:3; + uint32_t:1; + uint32_t mode:3; + uint32_t check_vmid:1; + uint32_t trap_id:3; + uint32_t:5; + uint32_t wave_id:4; + uint32_t simd_id:2; + uint32_t:2; + uint32_t queue_id:3; + uint32_t:1; + uint32_t vm_id:4; + } bitfields, bits; + uint32_t u32All; + signed int i32All; + float f32All; +}; + +union GRBM_GFX_INDEX_BITS { + struct { + uint32_t instance_index:8; + uint32_t sh_index:8; + uint32_t se_index:8; + uint32_t:5; + uint32_t sh_broadcast_writes:1; + uint32_t instance_broadcast_writes:1; + uint32_t se_broadcast_writes:1; + } bitfields, bits; + uint32_t u32All; + signed int i32All; + float f32All; +}; + /** * struct device_queue_manager_ops * @@ -56,7 +95,7 @@ struct device_process_node { * * @initialize: Initializes the pipelines and memory module for that device. * - * @start: Initializes the resources/modules the the device needs for queues + * @start: Initializes the resources/modules the device needs for queues * execution. This function is called on device initialization and after the * system woke up after suspension. * @@ -66,6 +105,12 @@ struct device_process_node { * @uninitialize: Destroys all the device queue manager resources allocated in * initialize routine. * + * @halt: This routine unmaps queues from runlist and set halt status to true + * so no more queues will be mapped to runlist until unhalt. + * + * @unhalt: This routine unset halt status to flase and maps queues back to + * runlist. + * * @create_kernel_queue: Creates kernel queue. Used for debug queue. * * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. @@ -77,23 +122,31 @@ struct device_process_node { * * @evict_process_queues: Evict all active queues of a process * - * @restore_process_queues: Restore all evicted queues queues of a process + * @restore_process_queues: Restore all evicted queues of a process * * @get_wave_state: Retrieves context save state and optionally copies the * control stack, if kept in the MQD, to the given userspace address. + * + * @reset_queues: reset queues which consume RAS poison + * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint. + * + * @checkpoint_mqd: checkpoint queue MQD contents for CRIU. */ struct device_queue_manager_ops { int (*create_queue)(struct device_queue_manager *dqm, struct queue *q, - struct qcm_process_device *qpd); + struct qcm_process_device *qpd, + const struct kfd_criu_queue_priv_data *qd, + const void *restore_mqd, + const void *restore_ctl_stack); int (*destroy_queue)(struct device_queue_manager *dqm, struct qcm_process_device *qpd, struct queue *q); int (*update_queue)(struct device_queue_manager *dqm, - struct queue *q); + struct queue *q, struct mqd_update_info *minfo); int (*register_process)(struct device_queue_manager *dqm, struct qcm_process_device *qpd); @@ -104,8 +157,9 @@ struct device_queue_manager_ops { int (*initialize)(struct device_queue_manager *dqm); int (*start)(struct device_queue_manager *dqm); int (*stop)(struct device_queue_manager *dqm); - void (*pre_reset)(struct device_queue_manager *dqm); void (*uninitialize)(struct device_queue_manager *dqm); + int (*halt)(struct device_queue_manager *dqm); + int (*unhalt)(struct device_queue_manager *dqm); int (*create_kernel_queue)(struct device_queue_manager *dqm, struct kernel_queue *kq, struct qcm_process_device *qpd); @@ -119,7 +173,8 @@ struct device_queue_manager_ops { enum cache_policy default_policy, enum cache_policy alternate_policy, void __user *alternate_aperture_base, - uint64_t alternate_aperture_size); + uint64_t alternate_aperture_size, + u32 misc_process_properties); int (*process_termination)(struct device_queue_manager *dqm, struct qcm_process_device *qpd); @@ -134,6 +189,17 @@ struct device_queue_manager_ops { void __user *ctl_stack, u32 *ctl_stack_used_size, u32 *save_area_used_size); + + int (*reset_queues)(struct device_queue_manager *dqm, + uint16_t pasid); + void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm, + const struct queue *q, u32 *mqd_size, + u32 *ctl_stack_size); + + int (*checkpoint_mqd)(struct device_queue_manager *dqm, + const struct queue *q, + void *mqd, + void *ctl_stack); }; struct device_queue_manager_asic_ops { @@ -144,12 +210,20 @@ struct device_queue_manager_asic_ops { enum cache_policy default_policy, enum cache_policy alternate_policy, void __user *alternate_aperture_base, - uint64_t alternate_aperture_size); + uint64_t alternate_aperture_size, + u32 misc_process_properties); void (*init_sdma_vm)(struct device_queue_manager *dqm, struct queue *q, struct qcm_process_device *qpd); struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type, - struct kfd_dev *dev); + struct kfd_node *dev); +}; + +struct dqm_detect_hang_info { + int pipe_id; + int queue_id; + int xcc_id; + uint64_t queue_address; }; /** @@ -169,8 +243,8 @@ struct device_queue_manager { struct device_queue_manager_asic_ops asic_ops; struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX]; - struct packet_manager packets; - struct kfd_dev *dev; + struct packet_manager packet_mgr; + struct kfd_node *dev; struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */ struct list_head queues; unsigned int saved_flags; @@ -181,8 +255,8 @@ struct device_queue_manager { unsigned int total_queue_count; unsigned int next_pipe_to_allocate; unsigned int *allocated_queues; - uint64_t sdma_bitmap; - uint64_t xgmi_sdma_bitmap; + DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES); + DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); /* the pasid mapping for each kfd vmid */ uint16_t vmid_pasid[VMID_NUM]; uint64_t pipelines_addr; @@ -191,26 +265,39 @@ struct device_queue_manager { struct kfd_mem_obj *fence_mem; bool active_runlist; int sched_policy; + uint32_t trap_debug_vmid; /* hw exception */ bool is_hws_hang; bool is_resetting; - struct work_struct hw_exception_work; struct kfd_mem_obj hiq_sdma_mqd; bool sched_running; + bool sched_halt; + + /* used for GFX 9.4.3 only */ + uint32_t current_logical_xcc_start; + + uint32_t wait_times; + + wait_queue_head_t destroy_wait; + + /* for per-queue reset support */ + struct dqm_detect_hang_info *detect_hang_info; + size_t detect_hang_info_size; + int detect_hang_count; }; void device_queue_manager_init_cik( struct device_queue_manager_asic_ops *asic_ops); -void device_queue_manager_init_cik_hawaii( - struct device_queue_manager_asic_ops *asic_ops); void device_queue_manager_init_vi( struct device_queue_manager_asic_ops *asic_ops); -void device_queue_manager_init_vi_tonga( - struct device_queue_manager_asic_ops *asic_ops); void device_queue_manager_init_v9( struct device_queue_manager_asic_ops *asic_ops); -void device_queue_manager_init_v10_navi10( +void device_queue_manager_init_v10( + struct device_queue_manager_asic_ops *asic_ops); +void device_queue_manager_init_v11( + struct device_queue_manager_asic_ops *asic_ops); +void device_queue_manager_init_v12( struct device_queue_manager_asic_ops *asic_ops); void program_sh_mem_settings(struct device_queue_manager *dqm, struct qcm_process_device *qpd); @@ -219,6 +306,27 @@ unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); +int reserve_debug_trap_vmid(struct device_queue_manager *dqm, + struct qcm_process_device *qpd); +int release_debug_trap_vmid(struct device_queue_manager *dqm, + struct qcm_process_device *qpd); +int suspend_queues(struct kfd_process *p, + uint32_t num_queues, + uint32_t grace_period, + uint64_t exception_clear_mask, + uint32_t *usr_queue_id_array); +int resume_queues(struct kfd_process *p, + uint32_t num_queues, + uint32_t *usr_queue_id_array); +void set_queue_snapshot_entry(struct queue *q, + uint64_t exception_clear_mask, + struct kfd_queue_snapshot_entry *qss_entry); +int debug_lock_and_unmap(struct device_queue_manager *dqm); +int debug_map_and_unlock(struct device_queue_manager *dqm); +int debug_refresh_runlist(struct device_queue_manager *dqm); +bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, + struct qcm_process_device *qpd, + int doorbell_off, u32 *queue_format); static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) { @@ -248,9 +356,17 @@ static inline void dqm_unlock(struct device_queue_manager *dqm) static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) { - /* - * SDMA activity counter is stored at queue's RPTR + 0x8 location. - */ + /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */ return get_user(*val, q_rptr + 1); } + +static inline void update_dqm_wait_times(struct device_queue_manager *dqm) +{ + if (dqm->dev->kfd2kgd->get_iq_wait_times) + dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev, + &dqm->wait_times, + ffs(dqm->dev->xcc_mask) - 1); +} + + #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ |