From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Iouri Tarassov Date: Tue, 18 Jan 2022 15:01:55 -0800 Subject: drivers: hv: dxgkrnl: Offer and reclaim allocations Implement ioctls to offer and reclaim compute device allocations: - LX_DXOFFERALLOCATIONS, - LX_DXRECLAIMALLOCATIONS2 When a user mode driver (UMD) does not need to access an allocation, it can "offer" it by issuing the LX_DXOFFERALLOCATIONS ioctl. This means that the allocation is not in use and its local device memory could be evicted. The freed space could be given to another allocation. When the allocation is again needed, the UMD can attempt to"reclaim" the allocation by issuing the LX_DXRECLAIMALLOCATIONS2 ioctl. If the allocation is still not evicted, the reclaim operation succeeds and no other action is required. If the reclaim operation fails, the caller must restore the content of the allocation before it can be used by the device. Signed-off-by: Iouri Tarassov [kms: Forward port to v6.1] Signed-off-by: Kelsey Steele --- drivers/hv/dxgkrnl/dxgkrnl.h | 8 + drivers/hv/dxgkrnl/dxgvmbus.c | 124 +++++++++- drivers/hv/dxgkrnl/dxgvmbus.h | 27 ++ drivers/hv/dxgkrnl/ioctl.c | 117 ++++++++- include/uapi/misc/d3dkmthk.h | 67 +++++ 5 files changed, 340 insertions(+), 3 deletions(-) diff --git a/drivers/hv/dxgkrnl/dxgkrnl.h b/drivers/hv/dxgkrnl/dxgkrnl.h index a55873bdd9a6..494ea8fb0bb3 100644 --- a/drivers/hv/dxgkrnl/dxgkrnl.h +++ b/drivers/hv/dxgkrnl/dxgkrnl.h @@ -865,6 +865,14 @@ int dxgvmb_send_set_allocation_priority(struct dxgprocess *process, int dxgvmb_send_get_allocation_priority(struct dxgprocess *process, struct dxgadapter *adapter, struct d3dkmt_getallocationpriority *a); +int dxgvmb_send_offer_allocations(struct dxgprocess *process, + struct dxgadapter *adapter, + struct d3dkmt_offerallocations *args); +int dxgvmb_send_reclaim_allocations(struct dxgprocess *process, + struct dxgadapter *adapter, + struct d3dkmthandle device, + struct d3dkmt_reclaimallocations2 *args, + u64 __user *paging_fence_value); int dxgvmb_send_change_vidmem_reservation(struct dxgprocess *process, struct dxgadapter *adapter, struct d3dkmthandle other_process, diff --git a/drivers/hv/dxgkrnl/dxgvmbus.c b/drivers/hv/dxgkrnl/dxgvmbus.c index 9a1864bb4e14..8448fd78975b 100644 --- a/drivers/hv/dxgkrnl/dxgvmbus.c +++ b/drivers/hv/dxgkrnl/dxgvmbus.c @@ -1858,7 +1858,7 @@ int dxgvmb_send_query_clock_calibration(struct dxgprocess *process, ret = copy_to_user(&inargs->clock_data, &result.clock_data, sizeof(result.clock_data)); if (ret) { - pr_err("%s failed to copy clock data", __func__); + DXG_ERR("failed to copy clock data"); ret = -EINVAL; goto cleanup; } @@ -2949,6 +2949,128 @@ int dxgvmb_send_get_allocation_priority(struct dxgprocess *process, return ret; } +int dxgvmb_send_offer_allocations(struct dxgprocess *process, + struct dxgadapter *adapter, + struct d3dkmt_offerallocations *args) +{ + struct dxgkvmb_command_offerallocations *command; + int ret = -EINVAL; + u32 alloc_size = sizeof(struct d3dkmthandle) * args->allocation_count; + u32 cmd_size = sizeof(struct dxgkvmb_command_offerallocations) + + alloc_size - sizeof(struct d3dkmthandle); + struct dxgvmbusmsg msg = {.hdr = NULL}; + + ret = init_message(&msg, adapter, process, cmd_size); + if (ret) + goto cleanup; + command = (void *)msg.msg; + + command_vgpu_to_host_init2(&command->hdr, + DXGK_VMBCOMMAND_OFFERALLOCATIONS, + process->host_handle); + command->flags = args->flags; + command->priority = args->priority; + command->device = args->device; + command->allocation_count = args->allocation_count; + if (args->resources) { + command->resources = true; + ret = copy_from_user(command->allocations, args->resources, + alloc_size); + } else { + ret = copy_from_user(command->allocations, + args->allocations, alloc_size); + } + if (ret) { + DXG_ERR("failed to copy input handles"); + ret = -EINVAL; + goto cleanup; + } + + ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size); + +cleanup: + free_message(&msg, process); + if (ret) + pr_debug("err: %s %d", __func__, ret); + return ret; +} + +int dxgvmb_send_reclaim_allocations(struct dxgprocess *process, + struct dxgadapter *adapter, + struct d3dkmthandle device, + struct d3dkmt_reclaimallocations2 *args, + u64 __user *paging_fence_value) +{ + struct dxgkvmb_command_reclaimallocations *command; + struct dxgkvmb_command_reclaimallocations_return *result; + int ret; + u32 alloc_size = sizeof(struct d3dkmthandle) * args->allocation_count; + u32 cmd_size = sizeof(struct dxgkvmb_command_reclaimallocations) + + alloc_size - sizeof(struct d3dkmthandle); + u32 result_size = sizeof(*result); + struct dxgvmbusmsgres msg = {.hdr = NULL}; + + if (args->results) + result_size += (args->allocation_count - 1) * + sizeof(enum d3dddi_reclaim_result); + + ret = init_message_res(&msg, adapter, process, cmd_size, result_size); + if (ret) + goto cleanup; + command = (void *)msg.msg; + result = msg.res; + + command_vgpu_to_host_init2(&command->hdr, + DXGK_VMBCOMMAND_RECLAIMALLOCATIONS, + process->host_handle); + command->device = device; + command->paging_queue = args->paging_queue; + command->allocation_count = args->allocation_count; + command->write_results = args->results != NULL; + if (args->resources) { + command->resources = true; + ret = copy_from_user(command->allocations, args->resources, + alloc_size); + } else { + ret = copy_from_user(command->allocations, + args->allocations, alloc_size); + } + if (ret) { + DXG_ERR("failed to copy input handles"); + ret = -EINVAL; + goto cleanup; + } + + ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size, + result, msg.res_size); + if (ret < 0) + goto cleanup; + ret = copy_to_user(paging_fence_value, + &result->paging_fence_value, sizeof(u64)); + if (ret) { + DXG_ERR("failed to copy paging fence"); + ret = -EINVAL; + goto cleanup; + } + + ret = ntstatus2int(result->status); + if (NT_SUCCESS(result->status) && args->results) { + ret = copy_to_user(args->results, result->discarded, + sizeof(result->discarded[0]) * + args->allocation_count); + if (ret) { + DXG_ERR("failed to copy results"); + ret = -EINVAL; + } + } + +cleanup: + free_message((struct dxgvmbusmsg *)&msg, process); + if (ret) + pr_debug("err: %s %d", __func__, ret); + return ret; +} + int dxgvmb_send_change_vidmem_reservation(struct dxgprocess *process, struct dxgadapter *adapter, struct d3dkmthandle other_process, diff --git a/drivers/hv/dxgkrnl/dxgvmbus.h b/drivers/hv/dxgkrnl/dxgvmbus.h index 17768ed0e68d..558c6576a262 100644 --- a/drivers/hv/dxgkrnl/dxgvmbus.h +++ b/drivers/hv/dxgkrnl/dxgvmbus.h @@ -653,6 +653,33 @@ struct dxgkvmb_command_markdeviceaserror { struct d3dkmt_markdeviceaserror args; }; +/* Returns ntstatus */ +struct dxgkvmb_command_offerallocations { + struct dxgkvmb_command_vgpu_to_host hdr; + struct d3dkmthandle device; + u32 allocation_count; + enum d3dkmt_offer_priority priority; + struct d3dkmt_offer_flags flags; + bool resources; + struct d3dkmthandle allocations[1]; +}; + +struct dxgkvmb_command_reclaimallocations { + struct dxgkvmb_command_vgpu_to_host hdr; + struct d3dkmthandle device; + struct d3dkmthandle paging_queue; + u32 allocation_count; + bool resources; + bool write_results; + struct d3dkmthandle allocations[1]; +}; + +struct dxgkvmb_command_reclaimallocations_return { + u64 paging_fence_value; + struct ntstatus status; + enum d3dddi_reclaim_result discarded[1]; +}; + /* Returns ntstatus */ struct dxgkvmb_command_changevideomemoryreservation { struct dxgkvmb_command_vgpu_to_host hdr; diff --git a/drivers/hv/dxgkrnl/ioctl.c b/drivers/hv/dxgkrnl/ioctl.c index 4babb21f38a9..fa880aa0196a 100644 --- a/drivers/hv/dxgkrnl/ioctl.c +++ b/drivers/hv/dxgkrnl/ioctl.c @@ -1961,6 +1961,119 @@ dxgkio_destroy_allocation(struct dxgprocess *process, void *__user inargs) return ret; } +static int +dxgkio_offer_allocations(struct dxgprocess *process, void *__user inargs) +{ + int ret; + struct d3dkmt_offerallocations args; + struct dxgdevice *device = NULL; + struct dxgadapter *adapter = NULL; + + ret = copy_from_user(&args, inargs, sizeof(args)); + if (ret) { + DXG_ERR("failed to copy input args"); + ret = -EINVAL; + goto cleanup; + } + + if (args.allocation_count > D3DKMT_MAKERESIDENT_ALLOC_MAX || + args.allocation_count == 0) { + DXG_ERR("invalid number of allocations"); + ret = -EINVAL; + goto cleanup; + } + + if ((args.resources == NULL) == (args.allocations == NULL)) { + DXG_ERR("invalid pointer to resources/allocations"); + ret = -EINVAL; + goto cleanup; + } + + device = dxgprocess_device_by_handle(process, args.device); + if (device == NULL) { + ret = -EINVAL; + goto cleanup; + } + + adapter = device->adapter; + ret = dxgadapter_acquire_lock_shared(adapter); + if (ret < 0) { + adapter = NULL; + goto cleanup; + } + + ret = dxgvmb_send_offer_allocations(process, adapter, &args); + +cleanup: + + if (adapter) + dxgadapter_release_lock_shared(adapter); + if (device) + kref_put(&device->device_kref, dxgdevice_release); + + DXG_TRACE("ioctl:%s %d", errorstr(ret), ret); + return ret; +} + +static int +dxgkio_reclaim_allocations(struct dxgprocess *process, void *__user inargs) +{ + int ret; + struct d3dkmt_reclaimallocations2 args; + struct dxgdevice *device = NULL; + struct dxgadapter *adapter = NULL; + struct d3dkmt_reclaimallocations2 * __user in_args = inargs; + + ret = copy_from_user(&args, inargs, sizeof(args)); + if (ret) { + DXG_ERR("failed to copy input args"); + ret = -EINVAL; + goto cleanup; + } + + if (args.allocation_count > D3DKMT_MAKERESIDENT_ALLOC_MAX || + args.allocation_count == 0) { + DXG_ERR("invalid number of allocations"); + ret = -EINVAL; + goto cleanup; + } + + if ((args.resources == NULL) == (args.allocations == NULL)) { + DXG_ERR("invalid pointer to resources/allocations"); + ret = -EINVAL; + goto cleanup; + } + + device = dxgprocess_device_by_object_handle(process, + HMGRENTRY_TYPE_DXGPAGINGQUEUE, + args.paging_queue); + if (device == NULL) { + ret = -EINVAL; + goto cleanup; + } + + adapter = device->adapter; + ret = dxgadapter_acquire_lock_shared(adapter); + if (ret < 0) { + adapter = NULL; + goto cleanup; + } + + ret = dxgvmb_send_reclaim_allocations(process, adapter, + device->handle, &args, + &in_args->paging_fence_value); + +cleanup: + + if (adapter) + dxgadapter_release_lock_shared(adapter); + if (device) + kref_put(&device->device_kref, dxgdevice_release); + + DXG_TRACE("ioctl:%s %d", errorstr(ret), ret); + return ret; +} + static int dxgkio_submit_command(struct dxgprocess *process, void *__user inargs) { @@ -4548,12 +4661,12 @@ static struct ioctl_desc ioctls[] = { /* 0x24 */ {}, /* 0x25 */ {dxgkio_lock2, LX_DXLOCK2}, /* 0x26 */ {dxgkio_mark_device_as_error, LX_DXMARKDEVICEASERROR}, -/* 0x27 */ {}, +/* 0x27 */ {dxgkio_offer_allocations, LX_DXOFFERALLOCATIONS}, /* 0x28 */ {}, /* 0x29 */ {}, /* 0x2a */ {dxgkio_query_alloc_residency, LX_DXQUERYALLOCATIONRESIDENCY}, /* 0x2b */ {}, -/* 0x2c */ {}, +/* 0x2c */ {dxgkio_reclaim_allocations, LX_DXRECLAIMALLOCATIONS2}, /* 0x2d */ {}, /* 0x2e */ {dxgkio_set_allocation_priority, LX_DXSETALLOCATIONPRIORITY}, /* 0x2f */ {}, diff --git a/include/uapi/misc/d3dkmthk.h b/include/uapi/misc/d3dkmthk.h index ea18242ceb83..46b9f6d303bf 100644 --- a/include/uapi/misc/d3dkmthk.h +++ b/include/uapi/misc/d3dkmthk.h @@ -61,6 +61,7 @@ struct winluid { #define D3DDDI_MAX_WRITTEN_PRIMARIES 16 #define D3DKMT_CREATEALLOCATION_MAX 1024 +#define D3DKMT_MAKERESIDENT_ALLOC_MAX (1024 * 10) #define D3DKMT_ADAPTERS_MAX 64 #define D3DDDI_MAX_BROADCAST_CONTEXT 64 #define D3DDDI_MAX_OBJECT_WAITED_ON 32 @@ -1087,6 +1088,68 @@ struct d3dddi_updateallocproperty { }; }; +enum d3dkmt_offer_priority { + _D3DKMT_OFFER_PRIORITY_LOW = 1, + _D3DKMT_OFFER_PRIORITY_NORMAL = 2, + _D3DKMT_OFFER_PRIORITY_HIGH = 3, + _D3DKMT_OFFER_PRIORITY_AUTO = 4, +}; + +struct d3dkmt_offer_flags { + union { + struct { + __u32 offer_immediately:1; + __u32 allow_decommit:1; + __u32 reserved:30; + }; + __u32 value; + }; +}; + +struct d3dkmt_offerallocations { + struct d3dkmthandle device; + __u32 reserved; +#ifdef __KERNEL__ + struct d3dkmthandle *resources; + const struct d3dkmthandle *allocations; +#else + __u64 resources; + __u64 allocations; +#endif + __u32 allocation_count; + enum d3dkmt_offer_priority priority; + struct d3dkmt_offer_flags flags; + __u32 reserved1; +}; + +enum d3dddi_reclaim_result { + _D3DDDI_RECLAIM_RESULT_OK = 0, + _D3DDDI_RECLAIM_RESULT_DISCARDED = 1, + _D3DDDI_RECLAIM_RESULT_NOT_COMMITTED = 2, +}; + +struct d3dkmt_reclaimallocations2 { + struct d3dkmthandle paging_queue; + __u32 allocation_count; +#ifdef __KERNEL__ + struct d3dkmthandle *resources; + struct d3dkmthandle *allocations; +#else + __u64 resources; + __u64 allocations; +#endif + union { +#ifdef __KERNEL__ + __u32 *discarded; + enum d3dddi_reclaim_result *results; +#else + __u64 discarded; + __u64 results; +#endif + }; + __u64 paging_fence_value; +}; + struct d3dkmt_changevideomemoryreservation { __u64 process; struct d3dkmthandle adapter; @@ -1360,8 +1423,12 @@ struct d3dkmt_shareobjectwithhost { _IOWR(0x47, 0x25, struct d3dkmt_lock2) #define LX_DXMARKDEVICEASERROR \ _IOWR(0x47, 0x26, struct d3dkmt_markdeviceaserror) +#define LX_DXOFFERALLOCATIONS \ + _IOWR(0x47, 0x27, struct d3dkmt_offerallocations) #define LX_DXQUERYALLOCATIONRESIDENCY \ _IOWR(0x47, 0x2a, struct d3dkmt_queryallocationresidency) +#define LX_DXRECLAIMALLOCATIONS2 \ + _IOWR(0x47, 0x2c, struct d3dkmt_reclaimallocations2) #define LX_DXSETALLOCATIONPRIORITY \ _IOWR(0x47, 0x2e, struct d3dkmt_setallocationpriority) #define LX_DXSIGNALSYNCHRONIZATIONOBJECTFROMCPU \ -- Armbian