[CRIU] [PATCH] Support dmabuf with amdgpu
Pavel Tikhomirov
ptikhomirov at virtuozzo.com
Mon Feb 3 07:26:14 MSK 2025
Thank you for you contribution! The mailing list is currently broken and
people might miss your contribution due to problems with delivering to
gmail addresses (at leat), so I've created a draft PR
https://github.com/checkpoint-restore/criu/pull/2581 here. (If you like,
feel free to create you own PR, and I will remove this one.)
Sorry for inconveniences.
On 2/1/25 03:00, David Francis wrote:
> This patch, in combination with an accompanying kernel patch,
> adds support for amdgpu dmabuf IPC with CRIU.
>
> It includes
> - Updates to the amdgpu_drm.h header file to match the kernel.
> - Inclusion of the kfd_ioctl. h header file to allow the plugin
> to call the new amdgpu CRIU ioctl.
> - Plugin file restore can now retry; new files_ext.c code to
> support this case.
> - amdgpu plugin now checks for shared bos during checkpoint
> by finding the gem handle for each bo; introduces
> dependency on 3-year-old libdrm change.
> - Unpause step is now its own callback (DUMP_DEVICE_LATE) instead
> of triggered by counting drm files.
> - amdgpu plugin restores bos by designating one process as the
> exporter; that process will acquire a dmabuf_fd for that
> bo and send it to the other processes to be imported.
> - amdgpu plugin tracks which bos it has restored; it will signal
> a retry if it needs to restore an imported bo but
> the corresponding export bo has not been restored.
> - New service_fd for transferring dmabuf fds. There is a function
> that plugins can call to send out a dmabuf fd, and a
> callback that will notify plugins that a dmabuf fd
> has been received over the socket.
> - New mechanism for finding fds to dup the received dmabuf fds to
> that won't conflict with other restore fds. Would like
> to unify this with find_unused_fd_pid.
>
> Signed-off-by: David Francis <David.Francis at amd.com>
> ---
> criu/cr-dump.c | 3 +
> criu/cr-restore.c | 1 +
> criu/files-ext.c | 10 +-
> criu/files.c | 113 ++
> criu/include/criu-plugin.h | 8 +-
> criu/include/files.h | 2 +
> criu/include/servicefd.h | 2 +
> criu/pie/restorer.c | 4 +
> criu/plugin.c | 1 +
> criu/servicefd.c | 12 +
> plugins/amdgpu/amdgpu_drm.h | 1638 +++++++++++++++++++++++++++
> plugins/amdgpu/amdgpu_plugin.c | 238 ++--
> plugins/amdgpu/amdgpu_plugin_drm.c | 368 +++++-
> plugins/amdgpu/amdgpu_plugin_drm.h | 9 +
> plugins/amdgpu/amdgpu_plugin_util.c | 157 ++-
> plugins/amdgpu/amdgpu_plugin_util.h | 49 +-
> plugins/amdgpu/criu-amdgpu.proto | 18 +
> plugins/amdgpu/kfd_ioctl.h | 1455 +++++++++++++++++++-----
> 18 files changed, 3700 insertions(+), 388 deletions(-)
> create mode 100644 plugins/amdgpu/amdgpu_drm.h
>
> diff --git a/criu/cr-dump.c b/criu/cr-dump.c
> index 1bc5d934f..34e756c7f 100644
> --- a/criu/cr-dump.c
> +++ b/criu/cr-dump.c
> @@ -2225,6 +2225,9 @@ int cr_dump_tasks(pid_t pid)
> goto err;
> }
>
> + if(run_plugins(DUMP_DEVICE_LATE, pid))
> + goto err;
> +
> if (parent_ie) {
> inventory_entry__free_unpacked(parent_ie, NULL);
> parent_ie = NULL;
> diff --git a/criu/cr-restore.c b/criu/cr-restore.c
> index ddca6b8ec..c7c74fc29 100644
> --- a/criu/cr-restore.c
> +++ b/criu/cr-restore.c
> @@ -3470,6 +3470,7 @@ static int sigreturn_restore(pid_t pid, struct task_restore_args *task_args, uns
> close_image_dir();
> close_proc();
> close_service_fd(TRANSPORT_FD_OFF);
> + close_service_fd(DMABUF_FD_OFF);
> close_service_fd(CR_PROC_FD_OFF);
> close_service_fd(ROOT_FD_OFF);
> close_service_fd(USERNSD_SK);
> diff --git a/criu/files-ext.c b/criu/files-ext.c
> index 95ec8e37c..4cc99d921 100644
> --- a/criu/files-ext.c
> +++ b/criu/files-ext.c
> @@ -45,10 +45,11 @@ static int open_fd(struct file_desc *d, int *new_fd)
> {
> struct ext_file_info *xfi;
> int fd;
> + bool retry_needed;
>
> xfi = container_of(d, struct ext_file_info, d);
>
> - fd = run_plugins(RESTORE_EXT_FILE, xfi->xfe->id);
> + fd = run_plugins(RESTORE_EXT_FILE, xfi->xfe->id, &retry_needed);
> if (fd < 0) {
> pr_err("Unable to restore %#x\n", xfi->xfe->id);
> return -1;
> @@ -57,8 +58,11 @@ static int open_fd(struct file_desc *d, int *new_fd)
> if (restore_fown(fd, xfi->xfe->fown))
> return -1;
>
> - *new_fd = fd;
> - return 0;
> + if (!retry_needed)
> + *new_fd = fd;
> + else
> + *new_fd = -1;
> + return retry_needed;
> }
>
> static struct file_desc_ops ext_desc_ops = {
> diff --git a/criu/files.c b/criu/files.c
> index 31e705bcc..5ca41e8c9 100644
> --- a/criu/files.c
> +++ b/criu/files.c
> @@ -62,6 +62,8 @@
> static struct hlist_head file_desc_hash[FDESC_HASH_SIZE];
> /* file_desc's, which fle is not owned by a process, that is able to open them */
> static LIST_HEAD(fake_master_head);
> +/* processes that have a file from a plugin that can use shared dmabuf_fds */
> +static LIST_HEAD(dmabuf_processes);
>
> static u32 max_file_desc_id = 0;
>
> @@ -831,10 +833,35 @@ static void collect_desc_fle(struct fdinfo_list_entry *new_le, struct file_desc
> }
> }
>
> +static int add_pid_to_dmabuf_list(int pid)
> +{
> + struct fdinfo_list_entry *le;
> +
> + list_for_each_entry(le, &dmabuf_processes, desc_list)
> + if (le->pid == pid)
> + return 0;
> +
> + le = alloc_fle(pid, NULL);
> +
> + if (!le)
> + return -ENOMEM;
> +
> + list_add(&le->desc_list, &dmabuf_processes);
> +
> + return 0;
> +}
> +
> struct fdinfo_list_entry *collect_fd_to(int pid, FdinfoEntry *e, struct rst_info *rst_info, struct file_desc *fdesc,
> bool fake, bool force_master)
> {
> struct fdinfo_list_entry *new_le;
> + int ret;
> +
> + if (fdesc->ops->type == FD_TYPES__EXT) {
> + ret = add_pid_to_dmabuf_list(pid);
> + if (ret)
> + return NULL;
> + }
>
> new_le = alloc_fle(pid, e);
> if (new_le) {
> @@ -983,6 +1010,14 @@ static void transport_name_gen(struct sockaddr_un *addr, int *len, int pid)
> *addr->sun_path = '\0';
> }
>
> +static void dmabuf_socket_name_gen(struct sockaddr_un *addr, int *len, int pid)
> +{
> + addr->sun_family = AF_UNIX;
> + snprintf(addr->sun_path, UNIX_PATH_MAX, "x/crtools-dmabuf-%d-%" PRIx64, pid, criu_run_id);
> + *len = SUN_LEN(addr);
> + *addr->sun_path = '\0';
> +}
> +
> static bool task_fle(struct pstree_item *task, struct fdinfo_list_entry *fle)
> {
> struct fdinfo_list_entry *tmp;
> @@ -1028,6 +1063,45 @@ static int recv_fd_from_peer(struct fdinfo_list_entry *fle)
> return 0;
> }
>
> +static int recv_dmabuf_fds(void)
> +{
> + int fd, newfd, ret, tsock, handle;
> +
> + tsock = get_service_fd(DMABUF_FD_OFF);
> +
> + while (true) {
> + ret = __recv_fds(tsock, &fd, 1, (void *)&handle, sizeof(handle), MSG_DONTWAIT);
> +
> + if (ret == -EAGAIN || ret == -EWOULDBLOCK)
> + return 1;
> + else if (ret)
> + return -1;
> +
> + newfd = get_unused_high_fd();
> +
> + reopen_fd_as(newfd, fd);
> +
> + run_plugins(DMABUF_FD, handle, newfd);
> + }
> +
> + return 0;
> +}
> +
> +static int send_dmabuf_fd_to_peer(int handle, int fd, int pid)
> +{
> + struct sockaddr_un saddr;
> + int len, sock, ret;
> +
> + sock = get_service_fd(DMABUF_FD_OFF);
> +
> + dmabuf_socket_name_gen(&saddr, &len, pid);
> + pr_info("\t\tSend dmabuf fd %d for handle %d to %s\n", fd, handle, saddr.sun_path + 1);
> + ret = send_fds(sock, &saddr, len, &fd, 1, (void *)&handle, sizeof(handle));
> + if (ret < 0)
> + return -1;
> + return set_fds_event(pid);
> +}
> +
> static int send_fd_to_peer(int fd, struct fdinfo_list_entry *fle)
> {
> struct sockaddr_un saddr;
> @@ -1132,6 +1206,25 @@ int setup_and_serve_out(struct fdinfo_list_entry *fle, int new_fd)
> return 0;
> }
>
> +int serve_out_dmabuf_fd(int handle, int fd)
> +{
> + int ret;
> + struct fdinfo_list_entry *fle;
> +
> + list_for_each_entry(fle, &dmabuf_processes, desc_list) {
> + ret = send_dmabuf_fd_to_peer(handle, fd, fle->pid);
> +
> + if (ret) {
> + pr_err("Can't sent fd %d to %d\n", fd, fle->pid);
> + goto out;
> + }
> + }
> +
> + ret = 0;
> +out:
> + return ret;
> +}
> +
> static int open_fd(struct fdinfo_list_entry *fle)
> {
> struct file_desc *d = fle->desc;
> @@ -1212,6 +1305,7 @@ static int open_fdinfos(struct pstree_item *me)
> do {
> progress = again = false;
> clear_fds_event();
> + recv_dmabuf_fds();
>
> list_for_each_entry_safe(fle, tmp, list, ps_list) {
> st = fle->stage;
> @@ -1707,6 +1801,25 @@ int open_transport_socket(void)
>
> if (install_service_fd(TRANSPORT_FD_OFF, sock) < 0)
> goto out;
> +
> +
> + sock = socket(PF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0);
> + if (sock < 0) {
> + pr_perror("Can't create socket");
> + goto out;
> + }
> +
> + dmabuf_socket_name_gen(&saddr, &slen, pid);
> + if (bind(sock, (struct sockaddr *)&saddr, slen) < 0) {
> + pr_perror("Can't bind dmabuf socket %s", saddr.sun_path + 1);
> + close(sock);
> + goto out;
> + }
> +
> + if (install_service_fd(DMABUF_FD_OFF, sock) < 0)
> + goto out;
> +
> +
> ret = 0;
> out:
> return ret;
> diff --git a/criu/include/criu-plugin.h b/criu/include/criu-plugin.h
> index 392ea9f53..7a6adb928 100644
> --- a/criu/include/criu-plugin.h
> +++ b/criu/include/criu-plugin.h
> @@ -60,6 +60,10 @@ enum {
>
> CR_PLUGIN_HOOK__CHECKPOINT_DEVICES = 11,
>
> + CR_PLUGIN_HOOK__DUMP_DEVICE_LATE = 12,
> +
> + CR_PLUGIN_HOOK__DMABUF_FD = 13,
> +
> CR_PLUGIN_HOOK__MAX
> };
>
> @@ -68,7 +72,7 @@ enum {
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__DUMP_UNIX_SK, int fd, int id);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__RESTORE_UNIX_SK, int id);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__DUMP_EXT_FILE, int fd, int id);
> -DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__RESTORE_EXT_FILE, int id);
> +DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__RESTORE_EXT_FILE, int id, bool *retry_needed);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__DUMP_EXT_MOUNT, char *mountpoint, int id);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__RESTORE_EXT_MOUNT, int id, char *mountpoint, char *old_root, int *is_file);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__DUMP_EXT_LINK, int index, int type, char *kind);
> @@ -78,6 +82,8 @@ DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__UPDATE_VMA_MAP, const char *path, const
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__RESUME_DEVICES_LATE, int pid);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__PAUSE_DEVICES, int pid);
> DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__CHECKPOINT_DEVICES, int pid);
> +DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__DUMP_DEVICE_LATE, int id);
> +DECLARE_PLUGIN_HOOK_ARGS(CR_PLUGIN_HOOK__DMABUF_FD, int handle, int fd);
>
> enum {
> CR_PLUGIN_STAGE__DUMP,
> diff --git a/criu/include/files.h b/criu/include/files.h
> index 31ebb0ca0..c3d8439df 100644
> --- a/criu/include/files.h
> +++ b/criu/include/files.h
> @@ -195,5 +195,7 @@ extern int open_transport_socket(void);
> extern int set_fds_event(pid_t virt);
> extern void wait_fds_event(void);
>
> +int serve_out_dmabuf_fd(int handle, int fd);
> +
> int find_unused_fd_pid(pid_t pid);
> #endif /* __CR_FILES_H__ */
> diff --git a/criu/include/servicefd.h b/criu/include/servicefd.h
> index 4265d94ed..780767013 100644
> --- a/criu/include/servicefd.h
> +++ b/criu/include/servicefd.h
> @@ -28,6 +28,7 @@ enum sfd_type {
> USERNSD_SK, /* Socket for usernsd */
> NS_FD_OFF, /* Node's net namespace fd */
> TRANSPORT_FD_OFF, /* to transfer file descriptors */
> + DMABUF_FD_OFF,
> RPC_SK_OFF,
> FDSTORE_SK_OFF,
>
> @@ -47,5 +48,6 @@ extern int install_service_fd(enum sfd_type type, int fd);
> extern int close_service_fd(enum sfd_type type);
> extern void __close_service_fd(enum sfd_type type);
> extern int clone_service_fd(struct pstree_item *me);
> +extern int get_unused_high_fd(void);
>
> #endif /* __CR_SERVICE_FD_H__ */
> diff --git a/criu/pie/restorer.c b/criu/pie/restorer.c
> index 6d048c3f1..3a56ed210 100644
> --- a/criu/pie/restorer.c
> +++ b/criu/pie/restorer.c
> @@ -1920,6 +1920,10 @@ __visible long __export_restore_task(struct task_restore_args *args)
>
> for (m = 0; m < sizeof(vma_entry->madv) * 8; m++) {
> if (vma_entry->madv & (1ul << m)) {
> +
> + if (!(vma_entry_is(vma_entry, VMA_AREA_REGULAR)))
> + continue;
> +
> ret = sys_madvise(vma_entry->start, vma_entry_len(vma_entry), m);
> if (ret) {
> pr_err("madvise(%" PRIx64 ", %" PRIu64 ", %ld) "
> diff --git a/criu/plugin.c b/criu/plugin.c
> index 65e79a069..965be2c11 100644
> --- a/criu/plugin.c
> +++ b/criu/plugin.c
> @@ -59,6 +59,7 @@ static cr_plugin_desc_t *cr_gen_plugin_desc(void *h, char *path)
> __assign_hook(RESUME_DEVICES_LATE, "cr_plugin_resume_devices_late");
> __assign_hook(PAUSE_DEVICES, "cr_plugin_pause_devices");
> __assign_hook(CHECKPOINT_DEVICES, "cr_plugin_checkpoint_devices");
> + __assign_hook(DUMP_DEVICE_LATE, "cr_plugin_dump_device_late");
>
> #undef __assign_hook
>
> diff --git a/criu/servicefd.c b/criu/servicefd.c
> index 06a8d3eba..9bd7c76a1 100644
> --- a/criu/servicefd.c
> +++ b/criu/servicefd.c
> @@ -25,6 +25,7 @@ int service_fd_rlim_cur;
>
> /* Base of current process service fds set */
> static int service_fd_base;
> +static int next_high_fd;
>
> /* Id of current process in shared fdt */
> static int service_fd_id = 0;
> @@ -53,6 +54,7 @@ const char *sfd_type_name(enum sfd_type type)
> [USERNSD_SK] = __stringify_1(USERNSD_SK),
> [NS_FD_OFF] = __stringify_1(NS_FD_OFF),
> [TRANSPORT_FD_OFF] = __stringify_1(TRANSPORT_FD_OFF),
> + [DMABUF_FD_OFF] = __stringify_1(DMABUF_FD_OFF),
> [RPC_SK_OFF] = __stringify_1(RPC_SK_OFF),
> [FDSTORE_SK_OFF] = __stringify_1(FDSTORE_SK_OFF),
> [SERVICE_FD_MAX] = __stringify_1(SERVICE_FD_MAX),
> @@ -312,5 +314,15 @@ int clone_service_fd(struct pstree_item *me)
> service_fd_id = id;
> ret = 0;
>
> + next_high_fd = service_fd_base + 1024;
> +
> return ret;
> }
> +
> +int get_unused_high_fd(void)
> +{
> + if (next_high_fd > service_fd_rlim_cur)
> + return -1;
> + next_high_fd += 1;
> + return next_high_fd - 1;
> +}
> diff --git a/plugins/amdgpu/amdgpu_drm.h b/plugins/amdgpu/amdgpu_drm.h
> new file mode 100644
> index 000000000..c6766fe5c
> --- /dev/null
> +++ b/plugins/amdgpu/amdgpu_drm.h
> @@ -0,0 +1,1638 @@
> +/* amdgpu_drm.h -- Public header for the amdgpu driver -*- linux-c -*-
> + *
> + * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
> + * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
> + * Copyright 2002 Tungsten Graphics, Inc., Cedar Park, Texas.
> + * Copyright 2014 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Authors:
> + * Kevin E. Martin <martin at valinux.com>
> + * Gareth Hughes <gareth at valinux.com>
> + * Keith Whitwell <keith at tungstengraphics.com>
> + */
> +
> +#ifndef __AMDGPU_DRM_H__
> +#define __AMDGPU_DRM_H__
> +
> +#include "drm.h"
> +
> +#if defined(__cplusplus)
> +extern "C" {
> +#endif
> +
> +#define DRM_AMDGPU_GEM_CREATE 0x00
> +#define DRM_AMDGPU_GEM_MMAP 0x01
> +#define DRM_AMDGPU_CTX 0x02
> +#define DRM_AMDGPU_BO_LIST 0x03
> +#define DRM_AMDGPU_CS 0x04
> +#define DRM_AMDGPU_INFO 0x05
> +#define DRM_AMDGPU_GEM_METADATA 0x06
> +#define DRM_AMDGPU_GEM_WAIT_IDLE 0x07
> +#define DRM_AMDGPU_GEM_VA 0x08
> +#define DRM_AMDGPU_WAIT_CS 0x09
> +#define DRM_AMDGPU_GEM_OP 0x10
> +#define DRM_AMDGPU_GEM_USERPTR 0x11
> +#define DRM_AMDGPU_WAIT_FENCES 0x12
> +#define DRM_AMDGPU_VM 0x13
> +#define DRM_AMDGPU_FENCE_TO_HANDLE 0x14
> +#define DRM_AMDGPU_SCHED 0x15
> +#define DRM_AMDGPU_USERQ 0x16
> +#define DRM_AMDGPU_USERQ_SIGNAL 0x17
> +#define DRM_AMDGPU_USERQ_WAIT 0x18
> +#define DRM_AMDGPU_CRIU_OP 0x19
> +
> +#define DRM_IOCTL_AMDGPU_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create)
> +#define DRM_IOCTL_AMDGPU_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap)
> +#define DRM_IOCTL_AMDGPU_CTX DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_CTX, union drm_amdgpu_ctx)
> +#define DRM_IOCTL_AMDGPU_BO_LIST DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_BO_LIST, union drm_amdgpu_bo_list)
> +#define DRM_IOCTL_AMDGPU_CS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_CS, union drm_amdgpu_cs)
> +#define DRM_IOCTL_AMDGPU_INFO DRM_IOW(DRM_COMMAND_BASE + DRM_AMDGPU_INFO, struct drm_amdgpu_info)
> +#define DRM_IOCTL_AMDGPU_GEM_METADATA DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_METADATA, struct drm_amdgpu_gem_metadata)
> +#define DRM_IOCTL_AMDGPU_GEM_WAIT_IDLE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_WAIT_IDLE, union drm_amdgpu_gem_wait_idle)
> +#define DRM_IOCTL_AMDGPU_GEM_VA DRM_IOW(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_VA, struct drm_amdgpu_gem_va)
> +#define DRM_IOCTL_AMDGPU_WAIT_CS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_WAIT_CS, union drm_amdgpu_wait_cs)
> +#define DRM_IOCTL_AMDGPU_GEM_OP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_OP, struct drm_amdgpu_gem_op)
> +#define DRM_IOCTL_AMDGPU_GEM_USERPTR DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_USERPTR, struct drm_amdgpu_gem_userptr)
> +#define DRM_IOCTL_AMDGPU_WAIT_FENCES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_WAIT_FENCES, union drm_amdgpu_wait_fences)
> +#define DRM_IOCTL_AMDGPU_VM DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_VM, union drm_amdgpu_vm)
> +#define DRM_IOCTL_AMDGPU_FENCE_TO_HANDLE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_FENCE_TO_HANDLE, union drm_amdgpu_fence_to_handle)
> +#define DRM_IOCTL_AMDGPU_SCHED DRM_IOW(DRM_COMMAND_BASE + DRM_AMDGPU_SCHED, union drm_amdgpu_sched)
> +#define DRM_IOCTL_AMDGPU_USERQ DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ, union drm_amdgpu_userq)
> +#define DRM_IOCTL_AMDGPU_USERQ_SIGNAL DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_SIGNAL, struct drm_amdgpu_userq_signal)
> +#define DRM_IOCTL_AMDGPU_USERQ_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait)
> +#define DRM_IOCTL_AMDGPU_CRIU_OP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_CRIU_OP, struct drm_amdgpu_criu_args)
> +
> +/**
> + * DOC: memory domains
> + *
> + * %AMDGPU_GEM_DOMAIN_CPU System memory that is not GPU accessible.
> + * Memory in this pool could be swapped out to disk if there is pressure.
> + *
> + * %AMDGPU_GEM_DOMAIN_GTT GPU accessible system memory, mapped into the
> + * GPU's virtual address space via gart. Gart memory linearizes non-contiguous
> + * pages of system memory, allows GPU access system memory in a linearized
> + * fashion.
> + *
> + * %AMDGPU_GEM_DOMAIN_VRAM Local video memory. For APUs, it is memory
> + * carved out by the BIOS.
> + *
> + * %AMDGPU_GEM_DOMAIN_GDS Global on-chip data storage used to share data
> + * across shader threads.
> + *
> + * %AMDGPU_GEM_DOMAIN_GWS Global wave sync, used to synchronize the
> + * execution of all the waves on a device.
> + *
> + * %AMDGPU_GEM_DOMAIN_OA Ordered append, used by 3D or Compute engines
> + * for appending data.
> + *
> + * %AMDGPU_GEM_DOMAIN_DOORBELL Doorbell. It is an MMIO region for
> + * signalling user mode queues.
> + */
> +#define AMDGPU_GEM_DOMAIN_CPU 0x1
> +#define AMDGPU_GEM_DOMAIN_GTT 0x2
> +#define AMDGPU_GEM_DOMAIN_VRAM 0x4
> +#define AMDGPU_GEM_DOMAIN_GDS 0x8
> +#define AMDGPU_GEM_DOMAIN_GWS 0x10
> +#define AMDGPU_GEM_DOMAIN_OA 0x20
> +#define AMDGPU_GEM_DOMAIN_DOORBELL 0x40
> +#define AMDGPU_GEM_DOMAIN_MASK (AMDGPU_GEM_DOMAIN_CPU | \
> + AMDGPU_GEM_DOMAIN_GTT | \
> + AMDGPU_GEM_DOMAIN_VRAM | \
> + AMDGPU_GEM_DOMAIN_GDS | \
> + AMDGPU_GEM_DOMAIN_GWS | \
> + AMDGPU_GEM_DOMAIN_OA | \
> + AMDGPU_GEM_DOMAIN_DOORBELL)
> +
> +/* Flag that CPU access will be required for the case of VRAM domain */
> +#define AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED (1 << 0)
> +/* Flag that CPU access will not work, this VRAM domain is invisible */
> +#define AMDGPU_GEM_CREATE_NO_CPU_ACCESS (1 << 1)
> +/* Flag that USWC attributes should be used for GTT */
> +#define AMDGPU_GEM_CREATE_CPU_GTT_USWC (1 << 2)
> +/* Flag that the memory should be in VRAM and cleared */
> +#define AMDGPU_GEM_CREATE_VRAM_CLEARED (1 << 3)
> +/* Flag that allocating the BO should use linear VRAM */
> +#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS (1 << 5)
> +/* Flag that BO is always valid in this VM */
> +#define AMDGPU_GEM_CREATE_VM_ALWAYS_VALID (1 << 6)
> +/* Flag that BO sharing will be explicitly synchronized */
> +#define AMDGPU_GEM_CREATE_EXPLICIT_SYNC (1 << 7)
> +/* Flag that indicates allocating MQD gart on GFX9, where the mtype
> + * for the second page onward should be set to NC. It should never
> + * be used by user space applications.
> + */
> +#define AMDGPU_GEM_CREATE_CP_MQD_GFX9 (1 << 8)
> +/* Flag that BO may contain sensitive data that must be wiped before
> + * releasing the memory
> + */
> +#define AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE (1 << 9)
> +/* Flag that BO will be encrypted and that the TMZ bit should be
> + * set in the PTEs when mapping this buffer via GPUVM or
> + * accessing it with various hw blocks
> + */
> +#define AMDGPU_GEM_CREATE_ENCRYPTED (1 << 10)
> +/* Flag that BO will be used only in preemptible context, which does
> + * not require GTT memory accounting
> + */
> +#define AMDGPU_GEM_CREATE_PREEMPTIBLE (1 << 11)
> +/* Flag that BO can be discarded under memory pressure without keeping the
> + * content.
> + */
> +#define AMDGPU_GEM_CREATE_DISCARDABLE (1 << 12)
> +/* Flag that BO is shared coherently between multiple devices or CPU threads.
> + * May depend on GPU instructions to flush caches to system scope explicitly.
> + *
> + * This influences the choice of MTYPE in the PTEs on GFXv9 and later GPUs and
> + * may override the MTYPE selected in AMDGPU_VA_OP_MAP.
> + */
> +#define AMDGPU_GEM_CREATE_COHERENT (1 << 13)
> +/* Flag that BO should not be cached by GPU. Coherent without having to flush
> + * GPU caches explicitly
> + *
> + * This influences the choice of MTYPE in the PTEs on GFXv9 and later GPUs and
> + * may override the MTYPE selected in AMDGPU_VA_OP_MAP.
> + */
> +#define AMDGPU_GEM_CREATE_UNCACHED (1 << 14)
> +/* Flag that BO should be coherent across devices when using device-level
> + * atomics. May depend on GPU instructions to flush caches to device scope
> + * explicitly, promoting them to system scope automatically.
> + *
> + * This influences the choice of MTYPE in the PTEs on GFXv9 and later GPUs and
> + * may override the MTYPE selected in AMDGPU_VA_OP_MAP.
> + */
> +#define AMDGPU_GEM_CREATE_EXT_COHERENT (1 << 15)
> +/* Set PTE.D and recompress during GTT->VRAM moves according to TILING flags. */
> +#define AMDGPU_GEM_CREATE_GFX12_DCC (1 << 16)
> +
> +struct drm_amdgpu_gem_create_in {
> + /** the requested memory size */
> + __u64 bo_size;
> + /** physical start_addr alignment in bytes for some HW requirements */
> + __u64 alignment;
> + /** the requested memory domains */
> + __u64 domains;
> + /** allocation flags */
> + __u64 domain_flags;
> +};
> +
> +struct drm_amdgpu_gem_create_out {
> + /** returned GEM object handle */
> + __u32 handle;
> + __u32 _pad;
> +};
> +
> +union drm_amdgpu_gem_create {
> + struct drm_amdgpu_gem_create_in in;
> + struct drm_amdgpu_gem_create_out out;
> +};
> +
> +/** Opcode to create new residency list. */
> +#define AMDGPU_BO_LIST_OP_CREATE 0
> +/** Opcode to destroy previously created residency list */
> +#define AMDGPU_BO_LIST_OP_DESTROY 1
> +/** Opcode to update resource information in the list */
> +#define AMDGPU_BO_LIST_OP_UPDATE 2
> +
> +struct drm_amdgpu_bo_list_in {
> + /** Type of operation */
> + __u32 operation;
> + /** Handle of list or 0 if we want to create one */
> + __u32 list_handle;
> + /** Number of BOs in list */
> + __u32 bo_number;
> + /** Size of each element describing BO */
> + __u32 bo_info_size;
> + /** Pointer to array describing BOs */
> + __u64 bo_info_ptr;
> +};
> +
> +struct drm_amdgpu_bo_list_entry {
> + /** Handle of BO */
> + __u32 bo_handle;
> + /** New (if specified) BO priority to be used during migration */
> + __u32 bo_priority;
> +};
> +
> +struct drm_amdgpu_bo_list_out {
> + /** Handle of resource list */
> + __u32 list_handle;
> + __u32 _pad;
> +};
> +
> +union drm_amdgpu_bo_list {
> + struct drm_amdgpu_bo_list_in in;
> + struct drm_amdgpu_bo_list_out out;
> +};
> +
> +/* context related */
> +#define AMDGPU_CTX_OP_ALLOC_CTX 1
> +#define AMDGPU_CTX_OP_FREE_CTX 2
> +#define AMDGPU_CTX_OP_QUERY_STATE 3
> +#define AMDGPU_CTX_OP_QUERY_STATE2 4
> +#define AMDGPU_CTX_OP_GET_STABLE_PSTATE 5
> +#define AMDGPU_CTX_OP_SET_STABLE_PSTATE 6
> +
> +/* GPU reset status */
> +#define AMDGPU_CTX_NO_RESET 0
> +/* this the context caused it */
> +#define AMDGPU_CTX_GUILTY_RESET 1
> +/* some other context caused it */
> +#define AMDGPU_CTX_INNOCENT_RESET 2
> +/* unknown cause */
> +#define AMDGPU_CTX_UNKNOWN_RESET 3
> +
> +/* indicate gpu reset occurred after ctx created */
> +#define AMDGPU_CTX_QUERY2_FLAGS_RESET (1<<0)
> +/* indicate vram lost occurred after ctx created */
> +#define AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST (1<<1)
> +/* indicate some job from this context once cause gpu hang */
> +#define AMDGPU_CTX_QUERY2_FLAGS_GUILTY (1<<2)
> +/* indicate some errors are detected by RAS */
> +#define AMDGPU_CTX_QUERY2_FLAGS_RAS_CE (1<<3)
> +#define AMDGPU_CTX_QUERY2_FLAGS_RAS_UE (1<<4)
> +/* indicate that the reset hasn't completed yet */
> +#define AMDGPU_CTX_QUERY2_FLAGS_RESET_IN_PROGRESS (1<<5)
> +
> +/* Context priority level */
> +#define AMDGPU_CTX_PRIORITY_UNSET -2048
> +#define AMDGPU_CTX_PRIORITY_VERY_LOW -1023
> +#define AMDGPU_CTX_PRIORITY_LOW -512
> +#define AMDGPU_CTX_PRIORITY_NORMAL 0
> +/*
> + * When used in struct drm_amdgpu_ctx_in, a priority above NORMAL requires
> + * CAP_SYS_NICE or DRM_MASTER
> +*/
> +#define AMDGPU_CTX_PRIORITY_HIGH 512
> +#define AMDGPU_CTX_PRIORITY_VERY_HIGH 1023
> +
> +/* select a stable profiling pstate for perfmon tools */
> +#define AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK 0xf
> +#define AMDGPU_CTX_STABLE_PSTATE_NONE 0
> +#define AMDGPU_CTX_STABLE_PSTATE_STANDARD 1
> +#define AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK 2
> +#define AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK 3
> +#define AMDGPU_CTX_STABLE_PSTATE_PEAK 4
> +
> +struct drm_amdgpu_ctx_in {
> + /** AMDGPU_CTX_OP_* */
> + __u32 op;
> + /** Flags */
> + __u32 flags;
> + __u32 ctx_id;
> + /** AMDGPU_CTX_PRIORITY_* */
> + __s32 priority;
> +};
> +
> +union drm_amdgpu_ctx_out {
> + struct {
> + __u32 ctx_id;
> + __u32 _pad;
> + } alloc;
> +
> + struct {
> + /** For future use, no flags defined so far */
> + __u64 flags;
> + /** Number of resets caused by this context so far. */
> + __u32 hangs;
> + /** Reset status since the last call of the ioctl. */
> + __u32 reset_status;
> + } state;
> +
> + struct {
> + __u32 flags;
> + __u32 _pad;
> + } pstate;
> +};
> +
> +union drm_amdgpu_ctx {
> + struct drm_amdgpu_ctx_in in;
> + union drm_amdgpu_ctx_out out;
> +};
> +
> +/* user queue IOCTL operations */
> +#define AMDGPU_USERQ_OP_CREATE 1
> +#define AMDGPU_USERQ_OP_FREE 2
> +
> +/*
> + * This structure is a container to pass input configuration
> + * info for all supported userqueue related operations.
> + * For operation AMDGPU_USERQ_OP_CREATE: user is expected
> + * to set all fields, excep the parameter 'queue_id'.
> + * For operation AMDGPU_USERQ_OP_FREE: the only input parameter expected
> + * to be set is 'queue_id', eveything else is ignored.
> + */
> +struct drm_amdgpu_userq_in {
> + /** AMDGPU_USERQ_OP_* */
> + __u32 op;
> + /** Queue id passed for operation USERQ_OP_FREE */
> + __u32 queue_id;
> + /** the target GPU engine to execute workload (AMDGPU_HW_IP_*) */
> + __u32 ip_type;
> + /**
> + * @doorbell_handle: the handle of doorbell GEM object
> + * associated with this userqueue client.
> + */
> + __u32 doorbell_handle;
> + /**
> + * @doorbell_offset: 32-bit offset of the doorbell in the doorbell bo.
> + * Kernel will generate absolute doorbell offset using doorbell_handle
> + * and doorbell_offset in the doorbell bo.
> + */
> + __u32 doorbell_offset;
> + __u32 _pad;
> + /**
> + * @queue_va: Virtual address of the GPU memory which holds the queue
> + * object. The queue holds the workload packets.
> + */
> + __u64 queue_va;
> + /**
> + * @queue_size: Size of the queue in bytes, this needs to be 256-byte
> + * aligned.
> + */
> + __u64 queue_size;
> + /**
> + * @rptr_va : Virtual address of the GPU memory which holds the ring RPTR.
> + * This object must be at least 8 byte in size and aligned to 8-byte offset.
> + */
> + __u64 rptr_va;
> + /**
> + * @wptr_va : Virtual address of the GPU memory which holds the ring WPTR.
> + * This object must be at least 8 byte in size and aligned to 8-byte offset.
> + *
> + * Queue, RPTR and WPTR can come from the same object, as long as the size
> + * and alignment related requirements are met.
> + */
> + __u64 wptr_va;
> + /**
> + * @mqd: MQD (memory queue descriptor) is a set of parameters which allow
> + * the GPU to uniquely define and identify a usermode queue.
> + *
> + * MQD data can be of different size for different GPU IP/engine and
> + * their respective versions/revisions, so this points to a __u64 *
> + * which holds IP specific MQD of this usermode queue.
> + */
> + __u64 mqd;
> + /**
> + * @size: size of MQD data in bytes, it must match the MQD structure
> + * size of the respective engine/revision defined in UAPI for ex, for
> + * gfx11 workloads, size = sizeof(drm_amdgpu_userq_mqd_gfx11).
> + */
> + __u64 mqd_size;
> +};
> +
> +/* The structure to carry output of userqueue ops */
> +struct drm_amdgpu_userq_out {
> + /**
> + * For operation AMDGPU_USERQ_OP_CREATE: This field contains a unique
> + * queue ID to represent the newly created userqueue in the system, otherwise
> + * it should be ignored.
> + */
> + __u32 queue_id;
> + __u32 _pad;
> +};
> +
> +union drm_amdgpu_userq {
> + struct drm_amdgpu_userq_in in;
> + struct drm_amdgpu_userq_out out;
> +};
> +
> +/* GFX V11 IP specific MQD parameters */
> +struct drm_amdgpu_userq_mqd_gfx11 {
> + /**
> + * @shadow_va: Virtual address of the GPU memory to hold the shadow buffer.
> + * Use AMDGPU_INFO_IOCTL to find the exact size of the object.
> + */
> + __u64 shadow_va;
> + /**
> + * @csa_va: Virtual address of the GPU memory to hold the CSA buffer.
> + * Use AMDGPU_INFO_IOCTL to find the exact size of the object.
> + */
> + __u64 csa_va;
> +};
> +
> +/* GFX V11 SDMA IP specific MQD parameters */
> +struct drm_amdgpu_userq_mqd_sdma_gfx11 {
> + /**
> + * @csa_va: Virtual address of the GPU memory to hold the CSA buffer.
> + * This must be a from a separate GPU object, and use AMDGPU_INFO IOCTL
> + * to get the size.
> + */
> + __u64 csa_va;
> +};
> +
> +/* GFX V11 Compute IP specific MQD parameters */
> +struct drm_amdgpu_userq_mqd_compute_gfx11 {
> + /**
> + * @eop_va: Virtual address of the GPU memory to hold the EOP buffer.
> + * This must be a from a separate GPU object, and use AMDGPU_INFO IOCTL
> + * to get the size.
> + */
> + __u64 eop_va;
> +};
> +
> +/* userq signal/wait ioctl */
> +struct drm_amdgpu_userq_signal {
> + /**
> + * @queue_id: Queue handle used by the userq fence creation function
> + * to retrieve the WPTR.
> + */
> + __u32 queue_id;
> + __u32 pad;
> + /**
> + * @syncobj_handles: The list of syncobj handles submitted by the user queue
> + * job to be signaled.
> + */
> + __u64 syncobj_handles;
> + /**
> + * @num_syncobj_handles: A count that represents the number of syncobj handles in
> + * @syncobj_handles.
> + */
> + __u64 num_syncobj_handles;
> + /**
> + * @bo_read_handles: The list of BO handles that the submitted user queue job
> + * is using for read only. This will update BO fences in the kernel.
> + */
> + __u64 bo_read_handles;
> + /**
> + * @bo_write_handles: The list of BO handles that the submitted user queue job
> + * is using for write only. This will update BO fences in the kernel.
> + */
> + __u64 bo_write_handles;
> + /**
> + * @num_bo_read_handles: A count that represents the number of read BO handles in
> + * @bo_read_handles.
> + */
> + __u32 num_bo_read_handles;
> + /**
> + * @num_bo_write_handles: A count that represents the number of write BO handles in
> + * @bo_write_handles.
> + */
> + __u32 num_bo_write_handles;
> +};
> +
> +struct drm_amdgpu_userq_fence_info {
> + /**
> + * @va: A gpu address allocated for each queue which stores the
> + * read pointer (RPTR) value.
> + */
> + __u64 va;
> + /**
> + * @value: A 64 bit value represents the write pointer (WPTR) of the
> + * queue commands which compared with the RPTR value to signal the
> + * fences.
> + */
> + __u64 value;
> +};
> +
> +struct drm_amdgpu_userq_wait {
> + /**
> + * @syncobj_handles: The list of syncobj handles submitted by the user queue
> + * job to get the va/value pairs.
> + */
> + __u64 syncobj_handles;
> + /**
> + * @syncobj_timeline_handles: The list of timeline syncobj handles submitted by
> + * the user queue job to get the va/value pairs at given @syncobj_timeline_points.
> + */
> + __u64 syncobj_timeline_handles;
> + /**
> + * @syncobj_timeline_points: The list of timeline syncobj points submitted by the
> + * user queue job for the corresponding @syncobj_timeline_handles.
> + */
> + __u64 syncobj_timeline_points;
> + /**
> + * @bo_read_handles: The list of read BO handles submitted by the user queue
> + * job to get the va/value pairs.
> + */
> + __u64 bo_read_handles;
> + /**
> + * @bo_write_handles: The list of write BO handles submitted by the user queue
> + * job to get the va/value pairs.
> + */
> + __u64 bo_write_handles;
> + /**
> + * @num_syncobj_timeline_handles: A count that represents the number of timeline
> + * syncobj handles in @syncobj_timeline_handles.
> + */
> + __u16 num_syncobj_timeline_handles;
> + /**
> + * @num_fences: This field can be used both as input and output. As input it defines
> + * the maximum number of fences that can be returned and as output it will specify
> + * how many fences were actually returned from the ioctl.
> + */
> + __u16 num_fences;
> + /**
> + * @num_syncobj_handles: A count that represents the number of syncobj handles in
> + * @syncobj_handles.
> + */
> + __u32 num_syncobj_handles;
> + /**
> + * @num_bo_read_handles: A count that represents the number of read BO handles in
> + * @bo_read_handles.
> + */
> + __u32 num_bo_read_handles;
> + /**
> + * @num_bo_write_handles: A count that represents the number of write BO handles in
> + * @bo_write_handles.
> + */
> + __u32 num_bo_write_handles;
> + /**
> + * @out_fences: The field is a return value from the ioctl containing the list of
> + * address/value pairs to wait for.
> + */
> + __u64 out_fences;
> +};
> +
> +/* vm ioctl */
> +#define AMDGPU_VM_OP_RESERVE_VMID 1
> +#define AMDGPU_VM_OP_UNRESERVE_VMID 2
> +
> +struct drm_amdgpu_vm_in {
> + /** AMDGPU_VM_OP_* */
> + __u32 op;
> + __u32 flags;
> +};
> +
> +struct drm_amdgpu_vm_out {
> + /** For future use, no flags defined so far */
> + __u64 flags;
> +};
> +
> +union drm_amdgpu_vm {
> + struct drm_amdgpu_vm_in in;
> + struct drm_amdgpu_vm_out out;
> +};
> +
> +/* sched ioctl */
> +#define AMDGPU_SCHED_OP_PROCESS_PRIORITY_OVERRIDE 1
> +#define AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE 2
> +
> +struct drm_amdgpu_sched_in {
> + /* AMDGPU_SCHED_OP_* */
> + __u32 op;
> + __u32 fd;
> + /** AMDGPU_CTX_PRIORITY_* */
> + __s32 priority;
> + __u32 ctx_id;
> +};
> +
> +union drm_amdgpu_sched {
> + struct drm_amdgpu_sched_in in;
> +};
> +
> +/*
> + * This is not a reliable API and you should expect it to fail for any
> + * number of reasons and have fallback path that do not use userptr to
> + * perform any operation.
> + */
> +#define AMDGPU_GEM_USERPTR_READONLY (1 << 0)
> +#define AMDGPU_GEM_USERPTR_ANONONLY (1 << 1)
> +#define AMDGPU_GEM_USERPTR_VALIDATE (1 << 2)
> +#define AMDGPU_GEM_USERPTR_REGISTER (1 << 3)
> +
> +struct drm_amdgpu_gem_userptr {
> + __u64 addr;
> + __u64 size;
> + /* AMDGPU_GEM_USERPTR_* */
> + __u32 flags;
> + /* Resulting GEM handle */
> + __u32 handle;
> +};
> +
> +/* SI-CI-VI: */
> +/* same meaning as the GB_TILE_MODE and GL_MACRO_TILE_MODE fields */
> +#define AMDGPU_TILING_ARRAY_MODE_SHIFT 0
> +#define AMDGPU_TILING_ARRAY_MODE_MASK 0xf
> +#define AMDGPU_TILING_PIPE_CONFIG_SHIFT 4
> +#define AMDGPU_TILING_PIPE_CONFIG_MASK 0x1f
> +#define AMDGPU_TILING_TILE_SPLIT_SHIFT 9
> +#define AMDGPU_TILING_TILE_SPLIT_MASK 0x7
> +#define AMDGPU_TILING_MICRO_TILE_MODE_SHIFT 12
> +#define AMDGPU_TILING_MICRO_TILE_MODE_MASK 0x7
> +#define AMDGPU_TILING_BANK_WIDTH_SHIFT 15
> +#define AMDGPU_TILING_BANK_WIDTH_MASK 0x3
> +#define AMDGPU_TILING_BANK_HEIGHT_SHIFT 17
> +#define AMDGPU_TILING_BANK_HEIGHT_MASK 0x3
> +#define AMDGPU_TILING_MACRO_TILE_ASPECT_SHIFT 19
> +#define AMDGPU_TILING_MACRO_TILE_ASPECT_MASK 0x3
> +#define AMDGPU_TILING_NUM_BANKS_SHIFT 21
> +#define AMDGPU_TILING_NUM_BANKS_MASK 0x3
> +
> +/* GFX9 - GFX11: */
> +#define AMDGPU_TILING_SWIZZLE_MODE_SHIFT 0
> +#define AMDGPU_TILING_SWIZZLE_MODE_MASK 0x1f
> +#define AMDGPU_TILING_DCC_OFFSET_256B_SHIFT 5
> +#define AMDGPU_TILING_DCC_OFFSET_256B_MASK 0xFFFFFF
> +#define AMDGPU_TILING_DCC_PITCH_MAX_SHIFT 29
> +#define AMDGPU_TILING_DCC_PITCH_MAX_MASK 0x3FFF
> +#define AMDGPU_TILING_DCC_INDEPENDENT_64B_SHIFT 43
> +#define AMDGPU_TILING_DCC_INDEPENDENT_64B_MASK 0x1
> +#define AMDGPU_TILING_DCC_INDEPENDENT_128B_SHIFT 44
> +#define AMDGPU_TILING_DCC_INDEPENDENT_128B_MASK 0x1
> +#define AMDGPU_TILING_SCANOUT_SHIFT 63
> +#define AMDGPU_TILING_SCANOUT_MASK 0x1
> +
> +/* GFX12 and later: */
> +#define AMDGPU_TILING_GFX12_SWIZZLE_MODE_SHIFT 0
> +#define AMDGPU_TILING_GFX12_SWIZZLE_MODE_MASK 0x7
> +/* These are DCC recompression setting for memory management: */
> +#define AMDGPU_TILING_GFX12_DCC_MAX_COMPRESSED_BLOCK_SHIFT 3
> +#define AMDGPU_TILING_GFX12_DCC_MAX_COMPRESSED_BLOCK_MASK 0x3 /* 0:64B, 1:128B, 2:256B */
> +#define AMDGPU_TILING_GFX12_DCC_NUMBER_TYPE_SHIFT 5
> +#define AMDGPU_TILING_GFX12_DCC_NUMBER_TYPE_MASK 0x7 /* CB_COLOR0_INFO.NUMBER_TYPE */
> +#define AMDGPU_TILING_GFX12_DCC_DATA_FORMAT_SHIFT 8
> +#define AMDGPU_TILING_GFX12_DCC_DATA_FORMAT_MASK 0x3f /* [0:4]:CB_COLOR0_INFO.FORMAT, [5]:MM */
> +
> +/* Set/Get helpers for tiling flags. */
> +#define AMDGPU_TILING_SET(field, value) \
> + (((__u64)(value) & AMDGPU_TILING_##field##_MASK) << AMDGPU_TILING_##field##_SHIFT)
> +#define AMDGPU_TILING_GET(value, field) \
> + (((__u64)(value) >> AMDGPU_TILING_##field##_SHIFT) & AMDGPU_TILING_##field##_MASK)
> +
> +#define AMDGPU_GEM_METADATA_OP_SET_METADATA 1
> +#define AMDGPU_GEM_METADATA_OP_GET_METADATA 2
> +
> +/** The same structure is shared for input/output */
> +struct drm_amdgpu_gem_metadata {
> + /** GEM Object handle */
> + __u32 handle;
> + /** Do we want get or set metadata */
> + __u32 op;
> + struct {
> + /** For future use, no flags defined so far */
> + __u64 flags;
> + /** family specific tiling info */
> + __u64 tiling_info;
> + __u32 data_size_bytes;
> + __u32 data[64];
> + } data;
> +};
> +
> +struct drm_amdgpu_gem_mmap_in {
> + /** the GEM object handle */
> + __u32 handle;
> + __u32 _pad;
> +};
> +
> +struct drm_amdgpu_gem_mmap_out {
> + /** mmap offset from the vma offset manager */
> + __u64 addr_ptr;
> +};
> +
> +union drm_amdgpu_gem_mmap {
> + struct drm_amdgpu_gem_mmap_in in;
> + struct drm_amdgpu_gem_mmap_out out;
> +};
> +
> +struct drm_amdgpu_gem_wait_idle_in {
> + /** GEM object handle */
> + __u32 handle;
> + /** For future use, no flags defined so far */
> + __u32 flags;
> + /** Absolute timeout to wait */
> + __u64 timeout;
> +};
> +
> +struct drm_amdgpu_gem_wait_idle_out {
> + /** BO status: 0 - BO is idle, 1 - BO is busy */
> + __u32 status;
> + /** Returned current memory domain */
> + __u32 domain;
> +};
> +
> +union drm_amdgpu_gem_wait_idle {
> + struct drm_amdgpu_gem_wait_idle_in in;
> + struct drm_amdgpu_gem_wait_idle_out out;
> +};
> +
> +struct drm_amdgpu_wait_cs_in {
> + /* Command submission handle
> + * handle equals 0 means none to wait for
> + * handle equals ~0ull means wait for the latest sequence number
> + */
> + __u64 handle;
> + /** Absolute timeout to wait */
> + __u64 timeout;
> + __u32 ip_type;
> + __u32 ip_instance;
> + __u32 ring;
> + __u32 ctx_id;
> +};
> +
> +struct drm_amdgpu_wait_cs_out {
> + /** CS status: 0 - CS completed, 1 - CS still busy */
> + __u64 status;
> +};
> +
> +union drm_amdgpu_wait_cs {
> + struct drm_amdgpu_wait_cs_in in;
> + struct drm_amdgpu_wait_cs_out out;
> +};
> +
> +struct drm_amdgpu_fence {
> + __u32 ctx_id;
> + __u32 ip_type;
> + __u32 ip_instance;
> + __u32 ring;
> + __u64 seq_no;
> +};
> +
> +struct drm_amdgpu_wait_fences_in {
> + /** This points to uint64_t * which points to fences */
> + __u64 fences;
> + __u32 fence_count;
> + __u32 wait_all;
> + __u64 timeout_ns;
> +};
> +
> +struct drm_amdgpu_wait_fences_out {
> + __u32 status;
> + __u32 first_signaled;
> +};
> +
> +union drm_amdgpu_wait_fences {
> + struct drm_amdgpu_wait_fences_in in;
> + struct drm_amdgpu_wait_fences_out out;
> +};
> +
> +#define AMDGPU_GEM_OP_GET_GEM_CREATE_INFO 0
> +#define AMDGPU_GEM_OP_SET_PLACEMENT 1
> +
> +/* Sets or returns a value associated with a buffer. */
> +struct drm_amdgpu_gem_op {
> + /** GEM object handle */
> + __u32 handle;
> + /** AMDGPU_GEM_OP_* */
> + __u32 op;
> + /** Input or return value */
> + __u64 value;
> +};
> +
> +#define AMDGPU_VA_OP_MAP 1
> +#define AMDGPU_VA_OP_UNMAP 2
> +#define AMDGPU_VA_OP_CLEAR 3
> +#define AMDGPU_VA_OP_REPLACE 4
> +
> +/* Delay the page table update till the next CS */
> +#define AMDGPU_VM_DELAY_UPDATE (1 << 0)
> +
> +/* Mapping flags */
> +/* readable mapping */
> +#define AMDGPU_VM_PAGE_READABLE (1 << 1)
> +/* writable mapping */
> +#define AMDGPU_VM_PAGE_WRITEABLE (1 << 2)
> +/* executable mapping, new for VI */
> +#define AMDGPU_VM_PAGE_EXECUTABLE (1 << 3)
> +/* partially resident texture */
> +#define AMDGPU_VM_PAGE_PRT (1 << 4)
> +/* MTYPE flags use bit 5 to 8 */
> +#define AMDGPU_VM_MTYPE_MASK (0xf << 5)
> +/* Default MTYPE. Pre-AI must use this. Recommended for newer ASICs. */
> +#define AMDGPU_VM_MTYPE_DEFAULT (0 << 5)
> +/* Use Non Coherent MTYPE instead of default MTYPE */
> +#define AMDGPU_VM_MTYPE_NC (1 << 5)
> +/* Use Write Combine MTYPE instead of default MTYPE */
> +#define AMDGPU_VM_MTYPE_WC (2 << 5)
> +/* Use Cache Coherent MTYPE instead of default MTYPE */
> +#define AMDGPU_VM_MTYPE_CC (3 << 5)
> +/* Use UnCached MTYPE instead of default MTYPE */
> +#define AMDGPU_VM_MTYPE_UC (4 << 5)
> +/* Use Read Write MTYPE instead of default MTYPE */
> +#define AMDGPU_VM_MTYPE_RW (5 << 5)
> +/* don't allocate MALL */
> +#define AMDGPU_VM_PAGE_NOALLOC (1 << 9)
> +
> +struct drm_amdgpu_gem_va {
> + /** GEM object handle */
> + __u32 handle;
> + __u32 _pad;
> + /** AMDGPU_VA_OP_* */
> + __u32 operation;
> + /** AMDGPU_VM_PAGE_* */
> + __u32 flags;
> + /** va address to assign . Must be correctly aligned.*/
> + __u64 va_address;
> + /** Specify offset inside of BO to assign. Must be correctly aligned.*/
> + __u64 offset_in_bo;
> + /** Specify mapping size. Must be correctly aligned. */
> + __u64 map_size;
> + /**
> + * vm_timeline_point is a sequence number used to add new timeline point.
> + */
> + __u64 vm_timeline_point;
> + /**
> + * The vm page table update fence is installed in given vm_timeline_syncobj_out
> + * at vm_timeline_point.
> + */
> + __u32 vm_timeline_syncobj_out;
> + /** the number of syncobj handles in @input_fence_syncobj_handles */
> + __u32 num_syncobj_handles;
> + /** Array of sync object handle to wait for given input fences */
> + __u64 input_fence_syncobj_handles;
> +};
> +
> +#define AMDGPU_HW_IP_GFX 0
> +#define AMDGPU_HW_IP_COMPUTE 1
> +#define AMDGPU_HW_IP_DMA 2
> +#define AMDGPU_HW_IP_UVD 3
> +#define AMDGPU_HW_IP_VCE 4
> +#define AMDGPU_HW_IP_UVD_ENC 5
> +#define AMDGPU_HW_IP_VCN_DEC 6
> +/*
> + * From VCN4, AMDGPU_HW_IP_VCN_ENC is re-used to support
> + * both encoding and decoding jobs.
> + */
> +#define AMDGPU_HW_IP_VCN_ENC 7
> +#define AMDGPU_HW_IP_VCN_JPEG 8
> +#define AMDGPU_HW_IP_VPE 9
> +#define AMDGPU_HW_IP_NUM 10
> +
> +#define AMDGPU_HW_IP_INSTANCE_MAX_COUNT 1
> +
> +#define AMDGPU_CHUNK_ID_IB 0x01
> +#define AMDGPU_CHUNK_ID_FENCE 0x02
> +#define AMDGPU_CHUNK_ID_DEPENDENCIES 0x03
> +#define AMDGPU_CHUNK_ID_SYNCOBJ_IN 0x04
> +#define AMDGPU_CHUNK_ID_SYNCOBJ_OUT 0x05
> +#define AMDGPU_CHUNK_ID_BO_HANDLES 0x06
> +#define AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES 0x07
> +#define AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT 0x08
> +#define AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL 0x09
> +#define AMDGPU_CHUNK_ID_CP_GFX_SHADOW 0x0a
> +
> +struct drm_amdgpu_cs_chunk {
> + __u32 chunk_id;
> + __u32 length_dw;
> + __u64 chunk_data;
> +};
> +
> +struct drm_amdgpu_cs_in {
> + /** Rendering context id */
> + __u32 ctx_id;
> + /** Handle of resource list associated with CS */
> + __u32 bo_list_handle;
> + __u32 num_chunks;
> + __u32 flags;
> + /** this points to __u64 * which point to cs chunks */
> + __u64 chunks;
> +};
> +
> +struct drm_amdgpu_cs_out {
> + __u64 handle;
> +};
> +
> +union drm_amdgpu_cs {
> + struct drm_amdgpu_cs_in in;
> + struct drm_amdgpu_cs_out out;
> +};
> +
> +/* Specify flags to be used for IB */
> +
> +/* This IB should be submitted to CE */
> +#define AMDGPU_IB_FLAG_CE (1<<0)
> +
> +/* Preamble flag, which means the IB could be dropped if no context switch */
> +#define AMDGPU_IB_FLAG_PREAMBLE (1<<1)
> +
> +/* Preempt flag, IB should set Pre_enb bit if PREEMPT flag detected */
> +#define AMDGPU_IB_FLAG_PREEMPT (1<<2)
> +
> +/* The IB fence should do the L2 writeback but not invalidate any shader
> + * caches (L2/vL1/sL1/I$). */
> +#define AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE (1 << 3)
> +
> +/* Set GDS_COMPUTE_MAX_WAVE_ID = DEFAULT before PACKET3_INDIRECT_BUFFER.
> + * This will reset wave ID counters for the IB.
> + */
> +#define AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID (1 << 4)
> +
> +/* Flag the IB as secure (TMZ)
> + */
> +#define AMDGPU_IB_FLAGS_SECURE (1 << 5)
> +
> +/* Tell KMD to flush and invalidate caches
> + */
> +#define AMDGPU_IB_FLAG_EMIT_MEM_SYNC (1 << 6)
> +
> +struct drm_amdgpu_cs_chunk_ib {
> + __u32 _pad;
> + /** AMDGPU_IB_FLAG_* */
> + __u32 flags;
> + /** Virtual address to begin IB execution */
> + __u64 va_start;
> + /** Size of submission */
> + __u32 ib_bytes;
> + /** HW IP to submit to */
> + __u32 ip_type;
> + /** HW IP index of the same type to submit to */
> + __u32 ip_instance;
> + /** Ring index to submit to */
> + __u32 ring;
> +};
> +
> +struct drm_amdgpu_cs_chunk_dep {
> + __u32 ip_type;
> + __u32 ip_instance;
> + __u32 ring;
> + __u32 ctx_id;
> + __u64 handle;
> +};
> +
> +struct drm_amdgpu_cs_chunk_fence {
> + __u32 handle;
> + __u32 offset;
> +};
> +
> +struct drm_amdgpu_cs_chunk_sem {
> + __u32 handle;
> +};
> +
> +struct drm_amdgpu_cs_chunk_syncobj {
> + __u32 handle;
> + __u32 flags;
> + __u64 point;
> +};
> +
> +#define AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ 0
> +#define AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD 1
> +#define AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD 2
> +
> +union drm_amdgpu_fence_to_handle {
> + struct {
> + struct drm_amdgpu_fence fence;
> + __u32 what;
> + __u32 pad;
> + } in;
> + struct {
> + __u32 handle;
> + } out;
> +};
> +
> +struct drm_amdgpu_cs_chunk_data {
> + union {
> + struct drm_amdgpu_cs_chunk_ib ib_data;
> + struct drm_amdgpu_cs_chunk_fence fence_data;
> + };
> +};
> +
> +#define AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW 0x1
> +
> +struct drm_amdgpu_cs_chunk_cp_gfx_shadow {
> + __u64 shadow_va;
> + __u64 csa_va;
> + __u64 gds_va;
> + __u64 flags;
> +};
> +
> +/*
> + * Query h/w info: Flag that this is integrated (a.h.a. fusion) GPU
> + *
> + */
> +#define AMDGPU_IDS_FLAGS_FUSION 0x1
> +#define AMDGPU_IDS_FLAGS_PREEMPTION 0x2
> +#define AMDGPU_IDS_FLAGS_TMZ 0x4
> +#define AMDGPU_IDS_FLAGS_CONFORMANT_TRUNC_COORD 0x8
> +
> +/* indicate if acceleration can be working */
> +#define AMDGPU_INFO_ACCEL_WORKING 0x00
> +/* get the crtc_id from the mode object id? */
> +#define AMDGPU_INFO_CRTC_FROM_ID 0x01
> +/* query hw IP info */
> +#define AMDGPU_INFO_HW_IP_INFO 0x02
> +/* query hw IP instance count for the specified type */
> +#define AMDGPU_INFO_HW_IP_COUNT 0x03
> +/* timestamp for GL_ARB_timer_query */
> +#define AMDGPU_INFO_TIMESTAMP 0x05
> +/* Query the firmware version */
> +#define AMDGPU_INFO_FW_VERSION 0x0e
> + /* Subquery id: Query VCE firmware version */
> + #define AMDGPU_INFO_FW_VCE 0x1
> + /* Subquery id: Query UVD firmware version */
> + #define AMDGPU_INFO_FW_UVD 0x2
> + /* Subquery id: Query GMC firmware version */
> + #define AMDGPU_INFO_FW_GMC 0x03
> + /* Subquery id: Query GFX ME firmware version */
> + #define AMDGPU_INFO_FW_GFX_ME 0x04
> + /* Subquery id: Query GFX PFP firmware version */
> + #define AMDGPU_INFO_FW_GFX_PFP 0x05
> + /* Subquery id: Query GFX CE firmware version */
> + #define AMDGPU_INFO_FW_GFX_CE 0x06
> + /* Subquery id: Query GFX RLC firmware version */
> + #define AMDGPU_INFO_FW_GFX_RLC 0x07
> + /* Subquery id: Query GFX MEC firmware version */
> + #define AMDGPU_INFO_FW_GFX_MEC 0x08
> + /* Subquery id: Query SMC firmware version */
> + #define AMDGPU_INFO_FW_SMC 0x0a
> + /* Subquery id: Query SDMA firmware version */
> + #define AMDGPU_INFO_FW_SDMA 0x0b
> + /* Subquery id: Query PSP SOS firmware version */
> + #define AMDGPU_INFO_FW_SOS 0x0c
> + /* Subquery id: Query PSP ASD firmware version */
> + #define AMDGPU_INFO_FW_ASD 0x0d
> + /* Subquery id: Query VCN firmware version */
> + #define AMDGPU_INFO_FW_VCN 0x0e
> + /* Subquery id: Query GFX RLC SRLC firmware version */
> + #define AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL 0x0f
> + /* Subquery id: Query GFX RLC SRLG firmware version */
> + #define AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM 0x10
> + /* Subquery id: Query GFX RLC SRLS firmware version */
> + #define AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM 0x11
> + /* Subquery id: Query DMCU firmware version */
> + #define AMDGPU_INFO_FW_DMCU 0x12
> + #define AMDGPU_INFO_FW_TA 0x13
> + /* Subquery id: Query DMCUB firmware version */
> + #define AMDGPU_INFO_FW_DMCUB 0x14
> + /* Subquery id: Query TOC firmware version */
> + #define AMDGPU_INFO_FW_TOC 0x15
> + /* Subquery id: Query CAP firmware version */
> + #define AMDGPU_INFO_FW_CAP 0x16
> + /* Subquery id: Query GFX RLCP firmware version */
> + #define AMDGPU_INFO_FW_GFX_RLCP 0x17
> + /* Subquery id: Query GFX RLCV firmware version */
> + #define AMDGPU_INFO_FW_GFX_RLCV 0x18
> + /* Subquery id: Query MES_KIQ firmware version */
> + #define AMDGPU_INFO_FW_MES_KIQ 0x19
> + /* Subquery id: Query MES firmware version */
> + #define AMDGPU_INFO_FW_MES 0x1a
> + /* Subquery id: Query IMU firmware version */
> + #define AMDGPU_INFO_FW_IMU 0x1b
> + /* Subquery id: Query VPE firmware version */
> + #define AMDGPU_INFO_FW_VPE 0x1c
> +
> +/* number of bytes moved for TTM migration */
> +#define AMDGPU_INFO_NUM_BYTES_MOVED 0x0f
> +/* the used VRAM size */
> +#define AMDGPU_INFO_VRAM_USAGE 0x10
> +/* the used GTT size */
> +#define AMDGPU_INFO_GTT_USAGE 0x11
> +/* Information about GDS, etc. resource configuration */
> +#define AMDGPU_INFO_GDS_CONFIG 0x13
> +/* Query information about VRAM and GTT domains */
> +#define AMDGPU_INFO_VRAM_GTT 0x14
> +/* Query information about register in MMR address space*/
> +#define AMDGPU_INFO_READ_MMR_REG 0x15
> +/* Query information about device: rev id, family, etc. */
> +#define AMDGPU_INFO_DEV_INFO 0x16
> +/* visible vram usage */
> +#define AMDGPU_INFO_VIS_VRAM_USAGE 0x17
> +/* number of TTM buffer evictions */
> +#define AMDGPU_INFO_NUM_EVICTIONS 0x18
> +/* Query memory about VRAM and GTT domains */
> +#define AMDGPU_INFO_MEMORY 0x19
> +/* Query vce clock table */
> +#define AMDGPU_INFO_VCE_CLOCK_TABLE 0x1A
> +/* Query vbios related information */
> +#define AMDGPU_INFO_VBIOS 0x1B
> + /* Subquery id: Query vbios size */
> + #define AMDGPU_INFO_VBIOS_SIZE 0x1
> + /* Subquery id: Query vbios image */
> + #define AMDGPU_INFO_VBIOS_IMAGE 0x2
> + /* Subquery id: Query vbios info */
> + #define AMDGPU_INFO_VBIOS_INFO 0x3
> +/* Query UVD handles */
> +#define AMDGPU_INFO_NUM_HANDLES 0x1C
> +/* Query sensor related information */
> +#define AMDGPU_INFO_SENSOR 0x1D
> + /* Subquery id: Query GPU shader clock */
> + #define AMDGPU_INFO_SENSOR_GFX_SCLK 0x1
> + /* Subquery id: Query GPU memory clock */
> + #define AMDGPU_INFO_SENSOR_GFX_MCLK 0x2
> + /* Subquery id: Query GPU temperature */
> + #define AMDGPU_INFO_SENSOR_GPU_TEMP 0x3
> + /* Subquery id: Query GPU load */
> + #define AMDGPU_INFO_SENSOR_GPU_LOAD 0x4
> + /* Subquery id: Query average GPU power */
> + #define AMDGPU_INFO_SENSOR_GPU_AVG_POWER 0x5
> + /* Subquery id: Query northbridge voltage */
> + #define AMDGPU_INFO_SENSOR_VDDNB 0x6
> + /* Subquery id: Query graphics voltage */
> + #define AMDGPU_INFO_SENSOR_VDDGFX 0x7
> + /* Subquery id: Query GPU stable pstate shader clock */
> + #define AMDGPU_INFO_SENSOR_STABLE_PSTATE_GFX_SCLK 0x8
> + /* Subquery id: Query GPU stable pstate memory clock */
> + #define AMDGPU_INFO_SENSOR_STABLE_PSTATE_GFX_MCLK 0x9
> + /* Subquery id: Query GPU peak pstate shader clock */
> + #define AMDGPU_INFO_SENSOR_PEAK_PSTATE_GFX_SCLK 0xa
> + /* Subquery id: Query GPU peak pstate memory clock */
> + #define AMDGPU_INFO_SENSOR_PEAK_PSTATE_GFX_MCLK 0xb
> + /* Subquery id: Query input GPU power */
> + #define AMDGPU_INFO_SENSOR_GPU_INPUT_POWER 0xc
> +/* Number of VRAM page faults on CPU access. */
> +#define AMDGPU_INFO_NUM_VRAM_CPU_PAGE_FAULTS 0x1E
> +#define AMDGPU_INFO_VRAM_LOST_COUNTER 0x1F
> +/* query ras mask of enabled features*/
> +#define AMDGPU_INFO_RAS_ENABLED_FEATURES 0x20
> +/* RAS MASK: UMC (VRAM) */
> +#define AMDGPU_INFO_RAS_ENABLED_UMC (1 << 0)
> +/* RAS MASK: SDMA */
> +#define AMDGPU_INFO_RAS_ENABLED_SDMA (1 << 1)
> +/* RAS MASK: GFX */
> +#define AMDGPU_INFO_RAS_ENABLED_GFX (1 << 2)
> +/* RAS MASK: MMHUB */
> +#define AMDGPU_INFO_RAS_ENABLED_MMHUB (1 << 3)
> +/* RAS MASK: ATHUB */
> +#define AMDGPU_INFO_RAS_ENABLED_ATHUB (1 << 4)
> +/* RAS MASK: PCIE */
> +#define AMDGPU_INFO_RAS_ENABLED_PCIE (1 << 5)
> +/* RAS MASK: HDP */
> +#define AMDGPU_INFO_RAS_ENABLED_HDP (1 << 6)
> +/* RAS MASK: XGMI */
> +#define AMDGPU_INFO_RAS_ENABLED_XGMI (1 << 7)
> +/* RAS MASK: DF */
> +#define AMDGPU_INFO_RAS_ENABLED_DF (1 << 8)
> +/* RAS MASK: SMN */
> +#define AMDGPU_INFO_RAS_ENABLED_SMN (1 << 9)
> +/* RAS MASK: SEM */
> +#define AMDGPU_INFO_RAS_ENABLED_SEM (1 << 10)
> +/* RAS MASK: MP0 */
> +#define AMDGPU_INFO_RAS_ENABLED_MP0 (1 << 11)
> +/* RAS MASK: MP1 */
> +#define AMDGPU_INFO_RAS_ENABLED_MP1 (1 << 12)
> +/* RAS MASK: FUSE */
> +#define AMDGPU_INFO_RAS_ENABLED_FUSE (1 << 13)
> +/* query video encode/decode caps */
> +#define AMDGPU_INFO_VIDEO_CAPS 0x21
> + /* Subquery id: Decode */
> + #define AMDGPU_INFO_VIDEO_CAPS_DECODE 0
> + /* Subquery id: Encode */
> + #define AMDGPU_INFO_VIDEO_CAPS_ENCODE 1
> +/* Query the max number of IBs per gang per submission */
> +#define AMDGPU_INFO_MAX_IBS 0x22
> +/* query last page fault info */
> +#define AMDGPU_INFO_GPUVM_FAULT 0x23
> +/* query FW object size and alignment */
> +#define AMDGPU_INFO_UQ_FW_AREAS 0x24
> +
> +#define AMDGPU_INFO_MMR_SE_INDEX_SHIFT 0
> +#define AMDGPU_INFO_MMR_SE_INDEX_MASK 0xff
> +#define AMDGPU_INFO_MMR_SH_INDEX_SHIFT 8
> +#define AMDGPU_INFO_MMR_SH_INDEX_MASK 0xff
> +
> +struct drm_amdgpu_query_fw {
> + /** AMDGPU_INFO_FW_* */
> + __u32 fw_type;
> + /**
> + * Index of the IP if there are more IPs of
> + * the same type.
> + */
> + __u32 ip_instance;
> + /**
> + * Index of the engine. Whether this is used depends
> + * on the firmware type. (e.g. MEC, SDMA)
> + */
> + __u32 index;
> + __u32 _pad;
> +};
> +
> +/* Input structure for the INFO ioctl */
> +struct drm_amdgpu_info {
> + /* Where the return value will be stored */
> + __u64 return_pointer;
> + /* The size of the return value. Just like "size" in "snprintf",
> + * it limits how many bytes the kernel can write. */
> + __u32 return_size;
> + /* The query request id. */
> + __u32 query;
> +
> + union {
> + struct {
> + __u32 id;
> + __u32 _pad;
> + } mode_crtc;
> +
> + struct {
> + /** AMDGPU_HW_IP_* */
> + __u32 type;
> + /**
> + * Index of the IP if there are more IPs of the same
> + * type. Ignored by AMDGPU_INFO_HW_IP_COUNT.
> + */
> + __u32 ip_instance;
> + } query_hw_ip;
> +
> + struct {
> + __u32 dword_offset;
> + /** number of registers to read */
> + __u32 count;
> + __u32 instance;
> + /** For future use, no flags defined so far */
> + __u32 flags;
> + } read_mmr_reg;
> +
> + struct drm_amdgpu_query_fw query_fw;
> +
> + struct {
> + __u32 type;
> + __u32 offset;
> + } vbios_info;
> +
> + struct {
> + __u32 type;
> + } sensor_info;
> +
> + struct {
> + __u32 type;
> + } video_cap;
> + };
> +};
> +
> +struct drm_amdgpu_info_gds {
> + /** GDS GFX partition size */
> + __u32 gds_gfx_partition_size;
> + /** GDS compute partition size */
> + __u32 compute_partition_size;
> + /** total GDS memory size */
> + __u32 gds_total_size;
> + /** GWS size per GFX partition */
> + __u32 gws_per_gfx_partition;
> + /** GSW size per compute partition */
> + __u32 gws_per_compute_partition;
> + /** OA size per GFX partition */
> + __u32 oa_per_gfx_partition;
> + /** OA size per compute partition */
> + __u32 oa_per_compute_partition;
> + __u32 _pad;
> +};
> +
> +struct drm_amdgpu_info_vram_gtt {
> + __u64 vram_size;
> + __u64 vram_cpu_accessible_size;
> + __u64 gtt_size;
> +};
> +
> +struct drm_amdgpu_heap_info {
> + /** max. physical memory */
> + __u64 total_heap_size;
> +
> + /** Theoretical max. available memory in the given heap */
> + __u64 usable_heap_size;
> +
> + /**
> + * Number of bytes allocated in the heap. This includes all processes
> + * and private allocations in the kernel. It changes when new buffers
> + * are allocated, freed, and moved. It cannot be larger than
> + * heap_size.
> + */
> + __u64 heap_usage;
> +
> + /**
> + * Theoretical possible max. size of buffer which
> + * could be allocated in the given heap
> + */
> + __u64 max_allocation;
> +};
> +
> +struct drm_amdgpu_memory_info {
> + struct drm_amdgpu_heap_info vram;
> + struct drm_amdgpu_heap_info cpu_accessible_vram;
> + struct drm_amdgpu_heap_info gtt;
> +};
> +
> +struct drm_amdgpu_info_firmware {
> + __u32 ver;
> + __u32 feature;
> +};
> +
> +struct drm_amdgpu_info_vbios {
> + __u8 name[64];
> + __u8 vbios_pn[64];
> + __u32 version;
> + __u32 pad;
> + __u8 vbios_ver_str[32];
> + __u8 date[32];
> +};
> +
> +#define AMDGPU_VRAM_TYPE_UNKNOWN 0
> +#define AMDGPU_VRAM_TYPE_GDDR1 1
> +#define AMDGPU_VRAM_TYPE_DDR2 2
> +#define AMDGPU_VRAM_TYPE_GDDR3 3
> +#define AMDGPU_VRAM_TYPE_GDDR4 4
> +#define AMDGPU_VRAM_TYPE_GDDR5 5
> +#define AMDGPU_VRAM_TYPE_HBM 6
> +#define AMDGPU_VRAM_TYPE_DDR3 7
> +#define AMDGPU_VRAM_TYPE_DDR4 8
> +#define AMDGPU_VRAM_TYPE_GDDR6 9
> +#define AMDGPU_VRAM_TYPE_DDR5 10
> +#define AMDGPU_VRAM_TYPE_LPDDR4 11
> +#define AMDGPU_VRAM_TYPE_LPDDR5 12
> +
> +struct drm_amdgpu_info_device {
> + /** PCI Device ID */
> + __u32 device_id;
> + /** Internal chip revision: A0, A1, etc.) */
> + __u32 chip_rev;
> + __u32 external_rev;
> + /** Revision id in PCI Config space */
> + __u32 pci_rev;
> + __u32 family;
> + __u32 num_shader_engines;
> + __u32 num_shader_arrays_per_engine;
> + /* in KHz */
> + __u32 gpu_counter_freq;
> + __u64 max_engine_clock;
> + __u64 max_memory_clock;
> + /* cu information */
> + __u32 cu_active_number;
> + /* NOTE: cu_ao_mask is INVALID, DON'T use it */
> + __u32 cu_ao_mask;
> + __u32 cu_bitmap[4][4];
> + /** Render backend pipe mask. One render backend is CB+DB. */
> + __u32 enabled_rb_pipes_mask;
> + __u32 num_rb_pipes;
> + __u32 num_hw_gfx_contexts;
> + /* PCIe version (the smaller of the GPU and the CPU/motherboard) */
> + __u32 pcie_gen;
> + __u64 ids_flags;
> + /** Starting virtual address for UMDs. */
> + __u64 virtual_address_offset;
> + /** The maximum virtual address */
> + __u64 virtual_address_max;
> + /** Required alignment of virtual addresses. */
> + __u32 virtual_address_alignment;
> + /** Page table entry - fragment size */
> + __u32 pte_fragment_size;
> + __u32 gart_page_size;
> + /** constant engine ram size*/
> + __u32 ce_ram_size;
> + /** video memory type info*/
> + __u32 vram_type;
> + /** video memory bit width*/
> + __u32 vram_bit_width;
> + /* vce harvesting instance */
> + __u32 vce_harvest_config;
> + /* gfx double offchip LDS buffers */
> + __u32 gc_double_offchip_lds_buf;
> + /* NGG Primitive Buffer */
> + __u64 prim_buf_gpu_addr;
> + /* NGG Position Buffer */
> + __u64 pos_buf_gpu_addr;
> + /* NGG Control Sideband */
> + __u64 cntl_sb_buf_gpu_addr;
> + /* NGG Parameter Cache */
> + __u64 param_buf_gpu_addr;
> + __u32 prim_buf_size;
> + __u32 pos_buf_size;
> + __u32 cntl_sb_buf_size;
> + __u32 param_buf_size;
> + /* wavefront size*/
> + __u32 wave_front_size;
> + /* shader visible vgprs*/
> + __u32 num_shader_visible_vgprs;
> + /* CU per shader array*/
> + __u32 num_cu_per_sh;
> + /* number of tcc blocks*/
> + __u32 num_tcc_blocks;
> + /* gs vgt table depth*/
> + __u32 gs_vgt_table_depth;
> + /* gs primitive buffer depth*/
> + __u32 gs_prim_buffer_depth;
> + /* max gs wavefront per vgt*/
> + __u32 max_gs_waves_per_vgt;
> + /* PCIe number of lanes (the smaller of the GPU and the CPU/motherboard) */
> + __u32 pcie_num_lanes;
> + /* always on cu bitmap */
> + __u32 cu_ao_bitmap[4][4];
> + /** Starting high virtual address for UMDs. */
> + __u64 high_va_offset;
> + /** The maximum high virtual address */
> + __u64 high_va_max;
> + /* gfx10 pa_sc_tile_steering_override */
> + __u32 pa_sc_tile_steering_override;
> + /* disabled TCCs */
> + __u64 tcc_disabled_mask;
> + __u64 min_engine_clock;
> + __u64 min_memory_clock;
> + /* The following fields are only set on gfx11+, older chips set 0. */
> + __u32 tcp_cache_size; /* AKA GL0, VMEM cache */
> + __u32 num_sqc_per_wgp;
> + __u32 sqc_data_cache_size; /* AKA SMEM cache */
> + __u32 sqc_inst_cache_size;
> + __u32 gl1c_cache_size;
> + __u32 gl2c_cache_size;
> + __u64 mall_size; /* AKA infinity cache */
> + /* high 32 bits of the rb pipes mask */
> + __u32 enabled_rb_pipes_mask_hi;
> + /* shadow area size for gfx11 */
> + __u32 shadow_size;
> + /* shadow area base virtual alignment for gfx11 */
> + __u32 shadow_alignment;
> + /* context save area size for gfx11 */
> + __u32 csa_size;
> + /* context save area base virtual alignment for gfx11 */
> + __u32 csa_alignment;
> +};
> +
> +struct drm_amdgpu_info_hw_ip {
> + /** Version of h/w IP */
> + __u32 hw_ip_version_major;
> + __u32 hw_ip_version_minor;
> + /** Capabilities */
> + __u64 capabilities_flags;
> + /** command buffer address start alignment*/
> + __u32 ib_start_alignment;
> + /** command buffer size alignment*/
> + __u32 ib_size_alignment;
> + /** Bitmask of available rings. Bit 0 means ring 0, etc. */
> + __u32 available_rings;
> + /** version info: bits 23:16 major, 15:8 minor, 7:0 revision */
> + __u32 ip_discovery_version;
> +};
> +
> +/* GFX metadata BO sizes and alignment info (in bytes) */
> +struct drm_amdgpu_info_uq_fw_areas_gfx {
> + /* shadow area size */
> + __u32 shadow_size;
> + /* shadow area base virtual mem alignment */
> + __u32 shadow_alignment;
> + /* context save area size */
> + __u32 csa_size;
> + /* context save area base virtual mem alignment */
> + __u32 csa_alignment;
> +};
> +
> +/* IP specific fw related information used in the
> + * subquery AMDGPU_INFO_UQ_FW_AREAS
> + */
> +struct drm_amdgpu_info_uq_fw_areas {
> + union {
> + struct drm_amdgpu_info_uq_fw_areas_gfx gfx;
> + };
> +};
> +
> +struct drm_amdgpu_info_num_handles {
> + /** Max handles as supported by firmware for UVD */
> + __u32 uvd_max_handles;
> + /** Handles currently in use for UVD */
> + __u32 uvd_used_handles;
> +};
> +
> +#define AMDGPU_VCE_CLOCK_TABLE_ENTRIES 6
> +
> +struct drm_amdgpu_info_vce_clock_table_entry {
> + /** System clock */
> + __u32 sclk;
> + /** Memory clock */
> + __u32 mclk;
> + /** VCE clock */
> + __u32 eclk;
> + __u32 pad;
> +};
> +
> +struct drm_amdgpu_info_vce_clock_table {
> + struct drm_amdgpu_info_vce_clock_table_entry entries[AMDGPU_VCE_CLOCK_TABLE_ENTRIES];
> + __u32 num_valid_entries;
> + __u32 pad;
> +};
> +
> +/* query video encode/decode caps */
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2 0
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4 1
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1 2
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC 3
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC 4
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG 5
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9 6
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1 7
> +#define AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_COUNT 8
> +
> +struct drm_amdgpu_info_video_codec_info {
> + __u32 valid;
> + __u32 max_width;
> + __u32 max_height;
> + __u32 max_pixels_per_frame;
> + __u32 max_level;
> + __u32 pad;
> +};
> +
> +struct drm_amdgpu_info_video_caps {
> + struct drm_amdgpu_info_video_codec_info codec_info[AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_COUNT];
> +};
> +
> +#define AMDGPU_VMHUB_TYPE_MASK 0xff
> +#define AMDGPU_VMHUB_TYPE_SHIFT 0
> +#define AMDGPU_VMHUB_TYPE_GFX 0
> +#define AMDGPU_VMHUB_TYPE_MM0 1
> +#define AMDGPU_VMHUB_TYPE_MM1 2
> +#define AMDGPU_VMHUB_IDX_MASK 0xff00
> +#define AMDGPU_VMHUB_IDX_SHIFT 8
> +
> +struct drm_amdgpu_info_gpuvm_fault {
> + __u64 addr;
> + __u32 status;
> + __u32 vmhub;
> +};
> +
> +struct drm_amdgpu_info_uq_metadata_gfx {
> + /* shadow area size for gfx11 */
> + __u32 shadow_size;
> + /* shadow area base virtual alignment for gfx11 */
> + __u32 shadow_alignment;
> + /* context save area size for gfx11 */
> + __u32 csa_size;
> + /* context save area base virtual alignment for gfx11 */
> + __u32 csa_alignment;
> +};
> +
> +struct drm_amdgpu_info_uq_metadata {
> + union {
> + struct drm_amdgpu_info_uq_metadata_gfx gfx;
> + };
> +};
> +
> +/*
> + * Supported GPU families
> + */
> +#define AMDGPU_FAMILY_UNKNOWN 0
> +#define AMDGPU_FAMILY_SI 110 /* Hainan, Oland, Verde, Pitcairn, Tahiti */
> +#define AMDGPU_FAMILY_CI 120 /* Bonaire, Hawaii */
> +#define AMDGPU_FAMILY_KV 125 /* Kaveri, Kabini, Mullins */
> +#define AMDGPU_FAMILY_VI 130 /* Iceland, Tonga */
> +#define AMDGPU_FAMILY_CZ 135 /* Carrizo, Stoney */
> +#define AMDGPU_FAMILY_AI 141 /* Vega10 */
> +#define AMDGPU_FAMILY_RV 142 /* Raven */
> +#define AMDGPU_FAMILY_NV 143 /* Navi10 */
> +#define AMDGPU_FAMILY_VGH 144 /* Van Gogh */
> +#define AMDGPU_FAMILY_GC_11_0_0 145 /* GC 11.0.0 */
> +#define AMDGPU_FAMILY_YC 146 /* Yellow Carp */
> +#define AMDGPU_FAMILY_GC_11_0_1 148 /* GC 11.0.1 */
> +#define AMDGPU_FAMILY_GC_10_3_6 149 /* GC 10.3.6 */
> +#define AMDGPU_FAMILY_GC_10_3_7 151 /* GC 10.3.7 */
> +#define AMDGPU_FAMILY_GC_11_5_0 150 /* GC 11.5.0 */
> +#define AMDGPU_FAMILY_GC_12_0_0 152 /* GC 12.0.0 */
> +
> +/* FIXME wrong namespace! */
> +struct drm_color_ctm_3x4 {
> + /*
> + * Conversion matrix with 3x4 dimensions in S31.32 sign-magnitude
> + * (not two's complement!) format.
> + */
> + __u64 matrix[12];
> +};
> +
> +/* CRIU ioctl
> + *
> + * When checkpointing a process, the CRIU amdgpu plugin will perform:
> + * 1. INFO op to get information about state that needs to be saved. This
> + * pauses execution until the checkpoint is done.
> + * 2. CHECKPOINT op to save state (BOs for now, TODO: CS contexts)
> + * 3. UNPAUSE op to resume execution when the checkpoint is done.
> + *
> + * When restoring a process, the CRIU amdgpu plugin will perform:
> + *
> + * 1. RESTORE op to restore state
> + * 2. RESUME op to restore userptr mappings (TODO)
> + */
> +enum drm_amdgpu_criu_op {
> + AMDGPU_CRIU_OP_PROCESS_INFO,
> + AMDGPU_CRIU_OP_CHECKPOINT,
> + AMDGPU_CRIU_OP_UNPAUSE,
> + AMDGPU_CRIU_OP_RESTORE,
> + AMDGPU_CRIU_OP_RESUME,
> +};
> +
> +struct drm_amdgpu_criu_args {
> + __u64 bos; /* user pointer to bos array */
> + __u64 priv_data; /* user pointer to private data */
> + __u64 priv_data_size;
> + __u32 num_bos;
> + __u32 num_objs;
> + __u32 pid;
> + __u32 op;
> + __u8 is_retry: 1;
> +};
> +
> +struct drm_amdgpu_criu_bo_bucket {
> + __u64 addr;
> + __u64 size;
> + __u64 offset;
> + __u64 restored_offset; /* During restore, updated offset for BO */
> + __u64 alloc_flags;
> + __u32 preferred_domains;
> + __u32 dmabuf_fd;
> + __u8 is_import: 1;
> + __u8 skip: 1;
> +};
> +
> +#if defined(__cplusplus)
> +}
> +#endif
> +
> +#endif
> diff --git a/plugins/amdgpu/amdgpu_plugin.c b/plugins/amdgpu/amdgpu_plugin.c
> index 96c086162..dd169c07c 100644
> --- a/plugins/amdgpu/amdgpu_plugin.c
> +++ b/plugins/amdgpu/amdgpu_plugin.c
> @@ -49,13 +49,6 @@ struct vma_metadata {
>
> /************************************ Global Variables ********************************************/
>
> -/**
> - * FD of KFD device used to checkpoint. On a multi-process
> - * tree the order of checkpointing goes from parent to child
> - * and so on - so saving the FD will not be overwritten
> - */
> -static int kfd_checkpoint_fd;
> -
> static LIST_HEAD(update_vma_info_list);
>
> size_t kfd_max_buffer_size;
> @@ -79,7 +72,7 @@ int kmtIoctl(int fd, unsigned long request, void *arg)
> /* In case pthread_atfork didn't catch it, this will
> * make any subsequent hsaKmt calls fail in CHECK_KFD_OPEN.
> */
> - pr_perror("KFD file descriptor not valid in this process");
> + pr_perror("KFD file descriptor not valid in this process: %d\n", fd);
> return ret;
> }
>
> @@ -110,7 +103,9 @@ static int allocate_device_entries(CriuKfd *e, int num_of_devices)
> }
>
> for (int i = 0; i < num_of_devices; i++) {
> - KfdDeviceEntry *entry = xzalloc(sizeof(*entry));
> + KfdDeviceEntry *entry;
> +
> + entry = xzalloc(sizeof(*entry));
>
> if (!entry) {
> pr_err("Failed to allocate entry\n");
> @@ -503,11 +498,11 @@ void free_and_unmap(uint64_t size, amdgpu_bo_handle h_bo, amdgpu_va_handle h_va,
> amdgpu_bo_free(h_bo);
> }
>
> -static int sdma_copy_bo(struct kfd_criu_bo_bucket bo_bucket, FILE *storage_fp,
> +int sdma_copy_bo(int shared_fd, uint64_t size, FILE *storage_fp,
> void *buffer, size_t buffer_size, amdgpu_device_handle h_dev,
> - uint64_t max_copy_size, enum sdma_op_type type)
> + uint64_t max_copy_size, enum sdma_op_type type, bool do_not_free)
> {
> - uint64_t size, src_bo_size, dst_bo_size, buffer_bo_size, bytes_remain, buffer_space_remain;
> + uint64_t src_bo_size, dst_bo_size, buffer_bo_size, bytes_remain, buffer_space_remain;
> uint64_t gpu_addr_src, gpu_addr_dst, gpu_addr_ib, copy_src, copy_dst, copy_size;
> amdgpu_va_handle h_va_src, h_va_dst, h_va_ib;
> amdgpu_bo_handle h_bo_src, h_bo_dst, h_bo_ib;
> @@ -520,10 +515,8 @@ static int sdma_copy_bo(struct kfd_criu_bo_bucket bo_bucket, FILE *storage_fp,
> uint32_t expired;
> amdgpu_context_handle h_ctx;
> uint32_t *ib = NULL;
> - int j, err, shared_fd, packets_per_buffer;
> + int j, err, packets_per_buffer;
>
> - shared_fd = bo_bucket.dmabuf_fd;
> - size = bo_bucket.size;
> buffer_bo_size = min(size, buffer_size);
> packets_per_buffer = ((buffer_bo_size - 1) / max_copy_size) + 1;
> src_bo_size = (type == SDMA_OP_VRAM_WRITE) ? buffer_bo_size : size;
> @@ -734,7 +727,8 @@ err_dst_bo_map:
> if (err)
> pr_perror("dest range free failed");
> err_dst_va:
> - err = amdgpu_bo_free(h_bo_dst);
> + if (!do_not_free)
> + err = amdgpu_bo_free(h_bo_dst);
> if (err)
> pr_perror("dest bo free failed");
> err_dst_bo_prep:
> @@ -822,8 +816,9 @@ void *dump_bo_contents(void *_thread_data)
> num_bos++;
>
> /* perform sDMA based vram copy */
> - ret = sdma_copy_bo(bo_buckets[i], bo_contents_fp, buffer, buffer_size, h_dev, max_copy_size,
> - SDMA_OP_VRAM_READ);
> + ret = sdma_copy_bo(bo_buckets[i].dmabuf_fd, bo_buckets[i].size, bo_contents_fp, buffer, buffer_size, h_dev, max_copy_size,
> + SDMA_OP_VRAM_READ, false);
> +
> if (ret) {
> pr_err("Failed to drain the BO using sDMA: bo_buckets[%d]\n", i);
> break;
> @@ -920,8 +915,8 @@ void *restore_bo_contents(void *_thread_data)
>
> num_bos++;
>
> - ret = sdma_copy_bo(bo_buckets[i], bo_contents_fp, buffer, buffer_size, h_dev, max_copy_size,
> - SDMA_OP_VRAM_WRITE);
> + ret = sdma_copy_bo(bo_buckets[i].dmabuf_fd, bo_buckets[i].size, bo_contents_fp, buffer, buffer_size, h_dev, max_copy_size,
> + SDMA_OP_VRAM_WRITE, false);
> if (ret) {
> pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n", i);
> break;
> @@ -1007,28 +1002,39 @@ int restore_hsakmt_shared_mem(const uint64_t shared_mem_size, const uint32_t sha
> return 0;
> }
>
> -static int unpause_process(int fd)
> +int amdgpu_unpause_processes(int pid)
> {
> int ret = 0;
> struct kfd_ioctl_criu_args args = { 0 };
> + struct list_head *l = get_dumped_fds();
> + struct dumped_fd *st;
> +
> + list_for_each_entry(st, l, l) {
> + if (st->is_drm) {
> + ret = amdgpu_plugin_drm_unpause_file(st->fd);
> + if (ret) {
> + pr_perror("Failed to unpause drm device file");
> + goto exit;
> + }
> + close(st->fd);
> + } else {
> + args.op = KFD_CRIU_OP_UNPAUSE;
>
> - args.op = KFD_CRIU_OP_UNPAUSE;
> -
> - ret = kmtIoctl(fd, AMDKFD_IOC_CRIU_OP, &args);
> - if (ret) {
> - pr_perror("Failed to unpause process");
> - goto exit;
> + ret = kmtIoctl(st->fd, AMDKFD_IOC_CRIU_OP, &args);
> + if (ret) {
> + pr_perror("Failed to unpause process");
> + goto exit;
> + }
> + }
> }
>
> - // Reset the KFD FD
> - kfd_checkpoint_fd = -1;
> - sys_close_drm_render_devices(&src_topology);
> -
> exit:
> pr_info("Process unpaused %s (ret:%d)\n", ret ? "Failed" : "Ok", ret);
> + clear_dumped_fds();
>
> return ret;
> }
> +CR_PLUGIN_REGISTER_HOOK(CR_PLUGIN_HOOK__DUMP_DEVICE_LATE, amdgpu_unpause_processes)
>
> static int save_devices(int fd, struct kfd_ioctl_criu_args *args, struct kfd_criu_device_bucket *device_buckets,
> CriuKfd *e)
> @@ -1072,6 +1078,8 @@ static int save_bos(int id, int fd, struct kfd_ioctl_criu_args *args, struct kfd
> {
> struct thread_data *thread_datas;
> int ret = 0, i;
> + amdgpu_device_handle h_dev;
> + uint32_t major, minor;
>
> pr_debug("Dumping %d BOs\n", args->num_bos);
>
> @@ -1095,6 +1103,21 @@ static int save_bos(int id, int fd, struct kfd_ioctl_criu_args *args, struct kfd
> boinfo->size = bo_bucket->size;
> boinfo->offset = bo_bucket->offset;
> boinfo->alloc_flags = bo_bucket->alloc_flags;
> +
> + ret = amdgpu_device_initialize(node_get_drm_render_device(sys_get_node_by_gpu_id(&src_topology, bo_bucket->gpu_id)), &major, &minor, &h_dev);
> +
> + boinfo->handle = get_gem_handle(h_dev, bo_bucket->dmabuf_fd);
> +
> + amdgpu_device_deinitialize(h_dev);
> +
> + boinfo->is_import = bo_bucket->is_import | shared_bo_has_exporter(boinfo->handle);
> + }
> + for (i = 0; i < e->num_of_bos; i++) {
> + KfdBoEntry *boinfo = e->bo_entries[i];
> +
> + ret = record_shared_bo(boinfo->handle, boinfo->is_import);
> + if (ret)
> + goto exit;
> }
>
> for (int i = 0; i < e->num_of_gpus; i++) {
> @@ -1215,11 +1238,10 @@ int amdgpu_plugin_dump_file(int fd, int id)
> return -1;
> }
>
> - /* Initialize number of device files that will be checkpointed */
> - init_gpu_count(&src_topology);
> -
> /* Check whether this plugin was called for kfd or render nodes */
> if (major(st.st_rdev) != major(st_kfd.st_rdev) || minor(st.st_rdev) != 0) {
> + char buffer[64];
> + char path[64];
>
> /* This is RenderD dumper plugin, for now just save renderD
> * minor number to be used during restore. In later phases this
> @@ -1229,11 +1251,11 @@ int amdgpu_plugin_dump_file(int fd, int id)
> if (ret)
> return ret;
>
> - /* Invoke unpause process if needed */
> - decrement_checkpoint_count();
> - if (checkpoint_is_complete()) {
> - ret = unpause_process(kfd_checkpoint_fd);
> - }
> + ret = record_dumped_fd(fd, true);
> + if (ret)
> + return ret;
> + sprintf(path, "/proc/self/fd/%d", fd);
> + readlink(path, buffer, 64);
>
> /* Need to return success here so that criu can call plugins for renderD nodes */
> return ret;
> @@ -1331,14 +1353,11 @@ int amdgpu_plugin_dump_file(int fd, int id)
>
> xfree(buf);
>
> -exit:
> - /* Restore all queues if conditions permit */
> - kfd_checkpoint_fd = fd;
> - decrement_checkpoint_count();
> - if (checkpoint_is_complete()) {
> - ret = unpause_process(fd);
> - }
> + ret = record_dumped_fd(fd, false);
> + if (ret)
> + goto exit;
>
> +exit:
> xfree((void *)args.devices);
> xfree((void *)args.bos);
> xfree((void *)args.priv_data);
> @@ -1409,6 +1428,7 @@ exit:
> static int restore_bos(struct kfd_ioctl_criu_args *args, CriuKfd *e)
> {
> struct kfd_criu_bo_bucket *bo_buckets;
> + bool retry_needed = false;
>
> pr_debug("Restoring %ld BOs\n", e->num_of_bos);
>
> @@ -1422,18 +1442,59 @@ static int restore_bos(struct kfd_ioctl_criu_args *args, CriuKfd *e)
> for (int i = 0; i < args->num_bos; i++) {
> struct kfd_criu_bo_bucket *bo_bucket = &bo_buckets[i];
> KfdBoEntry *bo_entry = e->bo_entries[i];
> + int dmabuf_fd = -1;
>
> - bo_bucket->gpu_id = bo_entry->gpu_id;
> bo_bucket->addr = bo_entry->addr;
> +
> + if (work_already_completed(bo_entry->handle, bo_entry->gpu_id)) {
> + bo_bucket->skip = 1;
> + } else if (bo_entry->handle != -1) {
> + if (bo_entry->is_import) {
> + dmabuf_fd = dmabuf_fd_for_handle(bo_entry->handle);
> + if (dmabuf_fd == -1) {
> + bo_bucket->skip = 1;
> + retry_needed = true;
> + }
> + }
> + }
> +
> + bo_bucket->is_import = bo_entry->is_import;
> +
> + bo_bucket->gpu_id = bo_entry->gpu_id;
> bo_bucket->size = bo_entry->size;
> bo_bucket->offset = bo_entry->offset;
> bo_bucket->alloc_flags = bo_entry->alloc_flags;
> + bo_bucket->dmabuf_fd = dmabuf_fd;
>
> plugin_log_msg("BO [%d] gpu_id:%x addr:%llx size:%llx offset:%llx\n", i, bo_bucket->gpu_id,
> bo_bucket->addr, bo_bucket->size, bo_bucket->offset);
> +
> }
>
> pr_info("Restore BOs Ok\n");
> +
> + return retry_needed;
> +}
> +
> +int save_vma_updates(uint64_t offset, uint64_t addr, uint64_t restored_offset, int fd)
> +{
> + struct vma_metadata *vma_md;
> +
> + vma_md = xmalloc(sizeof(*vma_md));
> + if (!vma_md) {
> + return -ENOMEM;
> + }
> +
> + memset(vma_md, 0, sizeof(*vma_md));
> +
> + vma_md->old_pgoff = offset;
> + vma_md->vma_entry = addr;
> +
> + vma_md->new_pgoff = restored_offset;
> + vma_md->fd = fd;
> +
> + list_add_tail(&vma_md->list, &update_vma_info_list);
> +
> return 0;
> }
>
> @@ -1452,6 +1513,9 @@ static int restore_bo_data(int id, struct kfd_criu_bo_bucket *bo_buckets, CriuKf
> struct kfd_criu_bo_bucket *bo_bucket = &bo_buckets[i];
> struct tp_node *tp_node;
>
> + if (bo_bucket->skip)
> + continue;
> +
> if (bo_bucket->alloc_flags & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT |
> KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP | KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)) {
> struct vma_metadata *vma_md;
> @@ -1481,10 +1545,6 @@ static int restore_bo_data(int id, struct kfd_criu_bo_bucket *bo_buckets, CriuKf
> vma_md->new_pgoff = bo_bucket->restored_offset;
> vma_md->fd = node_get_drm_render_device(tp_node);
>
> - plugin_log_msg("adding vma_entry:addr:0x%lx old-off:0x%lx "
> - "new_off:0x%lx new_minor:%d\n",
> - vma_md->vma_entry, vma_md->old_pgoff, vma_md->new_pgoff, vma_md->new_minor);
> -
> list_add_tail(&vma_md->list, &update_vma_info_list);
> }
> }
> @@ -1551,7 +1611,7 @@ exit:
> return ret;
> }
>
> -int amdgpu_plugin_restore_file(int id)
> +int amdgpu_plugin_restore_file(int id, bool *retry_needed)
> {
> int ret = 0, fd;
> char img_path[PATH_MAX];
> @@ -1562,6 +1622,8 @@ int amdgpu_plugin_restore_file(int id)
> size_t img_size;
> FILE *img_fp = NULL;
>
> + *retry_needed = false;
> +
> if (plugin_disabled)
> return -ENOTSUP;
>
> @@ -1609,8 +1671,6 @@ int amdgpu_plugin_restore_file(int id)
> }
> fclose(img_fp);
>
> - pr_info("render node gpu_id = 0x%04x\n", rd->gpu_id);
> -
> target_gpu_id = maps_get_dest_gpu(&restore_maps, rd->gpu_id);
> if (!target_gpu_id) {
> fd = -ENODEV;
> @@ -1626,8 +1686,18 @@ int amdgpu_plugin_restore_file(int id)
> pr_info("render node destination gpu_id = 0x%04x\n", tp_node->gpu_id);
>
> fd = node_get_drm_render_device(tp_node);
> - if (fd < 0)
> + if (fd < 0) {
> pr_err("Failed to open render device (minor:%d)\n", tp_node->drm_render_minor);
> + return -1;
> + }
> +
> + ret = amdgpu_plugin_drm_restore_file(fd, rd);
> + if (ret == 1)
> + *retry_needed = true;
> + if (ret < 0) {
> + fd = ret;
> + goto fail;
> + }
> fail:
> criu_render_node__free_unpacked(rd, NULL);
> xfree(buf);
> @@ -1639,12 +1709,19 @@ int amdgpu_plugin_restore_file(int id)
> * copy of the fd. CRIU core owns the duplicated returned fd, and amdgpu_plugin owns the fd stored in
> * tp_node.
> */
> - fd = dup(fd);
> - if (fd == -1) {
> - pr_perror("unable to duplicate the render fd");
> - return -1;
> +
> + if (fd < 0)
> + return fd;
> +
> + if (!(*retry_needed)) {
> + fd = dup(fd);
> + if (fd == -1) {
> + pr_perror("unable to duplicate the render fd");
> + return -1;
> + }
> + return fd;
> }
> - return fd;
> + return 0;
> }
>
> fd = open(AMDGPU_KFD_DEVICE, O_RDWR | O_CLOEXEC);
> @@ -1688,13 +1765,16 @@ int amdgpu_plugin_restore_file(int id)
> * This way, we know that the file descriptors we store will not conflict with file descriptors inside core
> * CRIU.
> */
> - fd_next = find_unused_fd_pid(e->pid);
> - if (fd_next <= 0) {
> - pr_err("Failed to find unused fd (fd:%d)\n", fd_next);
> - ret = -EINVAL;
> - goto exit;
> + if (fd_next == -1) {
> + fd_next = find_unused_fd_pid(e->pid);
> + if (fd_next <= 0) {
> + pr_err("Failed to find unused fd (fd:%d)\n", fd_next);
> + ret = -EINVAL;
> + goto exit;
> + }
> }
>
> +
> ret = devinfo_to_topology(e->device_entries, e->num_of_gpus + e->num_of_cpus, &src_topology);
> if (ret) {
> pr_err("Failed to convert stored device information to topology\n");
> @@ -1714,17 +1794,21 @@ int amdgpu_plugin_restore_file(int id)
> goto exit;
> }
>
> +
> ret = restore_devices(&args, e);
> if (ret)
> goto exit;
>
> ret = restore_bos(&args, e);
> - if (ret)
> + if (ret == 1)
> + *retry_needed = true;
> + else if (ret)
> goto exit;
>
> args.num_objects = e->num_of_objects;
> args.priv_data_size = e->priv_data.len;
> args.priv_data = (uintptr_t)e->priv_data.data;
> + args.is_retry = work_already_completed(-1, -1);
>
> args.op = KFD_CRIU_OP_RESTORE;
> if (kmtIoctl(fd, AMDKFD_IOC_CRIU_OP, &args) == -1) {
> @@ -1733,6 +1817,22 @@ int amdgpu_plugin_restore_file(int id)
> goto exit;
> }
>
> + ret = record_completed_work(-1, -1);
> + if (ret < 0)
> + goto exit;
> + for (int i = 0; i < args.num_bos; i++) {
> + struct kfd_criu_bo_bucket *bo_bucket = &((struct kfd_criu_bo_bucket *)args.bos)[i];
> + KfdBoEntry *bo_entry = e->bo_entries[i];
> +
> + if (!bo_bucket->skip && !work_already_completed(bo_entry->handle, bo_entry->gpu_id)) {
> + ret = record_completed_work(bo_entry->handle, bo_entry->gpu_id);
> + if (ret < 0)
> + goto exit;
> + if (!bo_entry->is_import)
> + serve_out_dmabuf_fd(bo_entry->handle, bo_bucket->dmabuf_fd);
> + }
> + }
> +
> ret = restore_bo_data(id, (struct kfd_criu_bo_bucket *)args.bos, e);
> if (ret)
> goto exit;
> @@ -1758,6 +1858,10 @@ exit:
> }
> CR_PLUGIN_REGISTER_HOOK(CR_PLUGIN_HOOK__RESTORE_EXT_FILE, amdgpu_plugin_restore_file)
>
> +int amdgpu_plugin_dmabuf_fd_update(int handle, int fd) {
> + return record_shared_dmabuf_fd(handle, fd);
> +}
> +CR_PLUGIN_REGISTER_HOOK(CR_PLUGIN_HOOK__DMABUF_FD, amdgpu_plugin_dmabuf_fd_update)
> /* return 0 if no match found
> * return -1 for error.
> * return 1 if vmap map must be adjusted.
> @@ -1857,6 +1961,8 @@ int amdgpu_plugin_resume_devices_late(int target_pid)
> }
> }
>
> + clear_completed_work_and_dmabuf_fds();
> +
> close(fd);
> return exit_code;
> }
> diff --git a/plugins/amdgpu/amdgpu_plugin_drm.c b/plugins/amdgpu/amdgpu_plugin_drm.c
> index d54cd937d..030392a85 100644
> --- a/plugins/amdgpu/amdgpu_plugin_drm.c
> +++ b/plugins/amdgpu/amdgpu_plugin_drm.c
> @@ -19,6 +19,7 @@
>
> #include <dirent.h>
> #include "common/list.h"
> +#include "files.h"
>
> #include "criu-amdgpu.pb-c.h"
>
> @@ -27,12 +28,79 @@
>
> #include "xmalloc.h"
> #include "criu-log.h"
> -#include "kfd_ioctl.h"
> +#include "amdgpu_drm.h"
> #include "amdgpu_plugin_drm.h"
> #include "amdgpu_plugin_util.h"
> #include "amdgpu_plugin_topology.h"
>
>
> +#include "util.h"
> +#include "common/scm.h"
> +
> +int get_gem_handle(amdgpu_device_handle h_dev, int dmabuf_fd)
> +{
> + uint32_t handle;
> + int fd = amdgpu_device_get_fd(h_dev);
> +
> + if (dmabuf_fd == -1) {
> + return -1;
> + }
> +
> + drmPrimeFDToHandle(fd, dmabuf_fd, &handle);
> +
> + return handle;
> +}
> +
> +int drmIoctl(int fd, unsigned long request, void *arg)
> +{
> + int ret, max_retries = 200;
> +
> + do {
> + ret = ioctl(fd, request, arg);
> + } while (ret == -1 && max_retries-- > 0 && (errno == EINTR || errno == EAGAIN));
> +
> + if (ret == -1 && errno == EBADF)
> + /* In case pthread_atfork didn't catch it, this will
> + * make any subsequent hsaKmt calls fail in CHECK_KFD_OPEN.
> + */
> + pr_perror("KFD file descriptor not valid in this process");
> + return ret;
> +}
> +
> +static int allocate_bo_entries(CriuRenderNode *e, int num_bos)
> +{
> + e->bo_entries = xmalloc(sizeof(DrmBoEntry *) * num_bos);
> + if (!e->bo_entries) {
> + pr_err("Failed to allocate bo_info\n");
> + return -ENOMEM;
> + }
> +
> + for (int i = 0; i < num_bos; i++) {
> + DrmBoEntry *entry = xzalloc(sizeof(*entry));
> +
> + if (!entry) {
> + pr_err("Failed to allocate botest\n");
> + return -ENOMEM;
> + }
> +
> + drm_bo_entry__init(entry);
> +
> + e->bo_entries[i] = entry;
> + e->n_bo_entries++;
> + }
> + return 0;
> +}
> +
> +static void free_e(CriuRenderNode *e)
> +{
> + for (int i = 0; i < e->n_bo_entries; i++) {
> + if (e->bo_entries[i])
> + xfree(e->bo_entries[i]);
> + }
> +
> + xfree(e);
> +}
> +
> int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *st)
> {
> char path[PATH_MAX];
> @@ -60,19 +128,195 @@ int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *st)
> return 0;
> }
>
> +static int restore_bo_contents_drm(int drm_render_minor, pid_t pid, int drm_fd, uint64_t num_of_bos, struct drm_amdgpu_criu_bo_bucket *bo_buckets)
> +{
> + size_t image_size = 0, total_bo_size = 0, max_bo_size = 0, buffer_size;
> + struct amdgpu_gpu_info gpu_info = { 0 };
> + amdgpu_device_handle h_dev;
> + uint64_t max_copy_size;
> + uint32_t major, minor;
> + FILE *bo_contents_fp = NULL;
> + void *buffer = NULL;
> + char img_path[40];
> + int num_bos = 0;
> + int i, ret = 0;
> +
> + ret = amdgpu_device_initialize(drm_fd, &major, &minor, &h_dev);
> + if (ret) {
> + pr_perror("failed to initialize device");
> + goto exit;
> + }
> + plugin_log_msg("libdrm initialized successfully\n");
> +
> + ret = amdgpu_query_gpu_info(h_dev, &gpu_info);
> + if (ret) {
> + pr_perror("failed to query gpuinfo via libdrm");
> + goto exit;
> + }
> +
> + max_copy_size = (gpu_info.family_id >= AMDGPU_FAMILY_AI) ? SDMA_LINEAR_COPY_MAX_SIZE :
> + SDMA_LINEAR_COPY_MAX_SIZE - 1;
> +
> + for (i = 0; i < num_of_bos; i++) {
> + if (bo_buckets[i].preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
> + total_bo_size += bo_buckets[i].size;
> +
> + if (bo_buckets[i].size > max_bo_size)
> + max_bo_size = bo_buckets[i].size;
> + }
> + }
> +
> + buffer_size = max_bo_size;
> +
> + posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), buffer_size);
> + if (!buffer) {
> + pr_perror("Failed to alloc aligned memory. Consider setting KFD_MAX_BUFFER_SIZE.");
> + ret = -ENOMEM;
> + goto exit;
> + }
> +
> + for (i = 0; i < num_of_bos; i++) {
> +
> + if (!(bo_buckets[i].preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)))
> + continue;
> +
> + if (bo_buckets[i].addr == -1)
> + continue;
> +
> + num_bos++;
> +
> + snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, pid, drm_render_minor, i);
> + bo_contents_fp = open_img_file(img_path, false, &image_size);
> +
> + ret = sdma_copy_bo(bo_buckets[i].dmabuf_fd, bo_buckets[i].size, bo_contents_fp, buffer, buffer_size, h_dev, max_copy_size,
> + SDMA_OP_VRAM_WRITE, true);
> + if (ret) {
> + pr_err("Failed to fill the BO using sDMA: bo_buckets[%d]\n", i);
> + break;
> + }
> + plugin_log_msg("** Successfully filled the BO using sDMA: bo_buckets[%d] **\n", i);
> +
> + if (bo_contents_fp)
> + fclose(bo_contents_fp);
> + }
> +
> +exit:
> + for (int i = 0; i < num_of_bos; i++) {
> + if (bo_buckets[i].dmabuf_fd != KFD_INVALID_FD)
> + close(bo_buckets[i].dmabuf_fd);
> + }
> +
> + xfree(buffer);
> +
> + amdgpu_device_deinitialize(h_dev);
> + return ret;
> +}
>
> int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
> {
> - CriuRenderNode rd = CRIU_RENDER_NODE__INIT;
> - struct tp_node *tp_node;
> + CriuRenderNode *rd = NULL;
> char path[PATH_MAX];
> unsigned char *buf;
> int minor;
> int len;
> int ret;
> + struct drm_amdgpu_criu_args args = {0};
> + size_t image_size;
> + struct tp_node *tp_node;
> +
> + rd = xmalloc(sizeof(*rd));
> + if (!rd) {
> + ret = -ENOMEM;
> + goto exit;
> + }
> + criu_render_node__init(rd);
>
> /* Get the topology node of the DRM device */
> minor = minor(drm->st_rdev);
> + rd->drm_render_minor = minor;
> +
> + args.op = AMDGPU_CRIU_OP_PROCESS_INFO;
> + if (drmIoctl(fd, DRM_IOCTL_AMDGPU_CRIU_OP, &args) == -1) {
> + pr_perror("Failed to call process info ioctl");
> + ret = -1;
> + goto exit;
> + }
> +
> + rd->pid = args.pid;
> + rd->num_of_bos = args.num_bos;
> + rd->num_of_objects = args.num_objs;
> + ret = allocate_bo_entries(rd, args.num_bos);
> + if (ret)
> + goto exit;
> +
> + args.bos = (uintptr_t)xzalloc((args.num_bos * sizeof(struct drm_amdgpu_criu_bo_bucket)));
> + if (!args.bos) {
> + ret = -ENOMEM;
> + goto exit;
> + }
> +
> + args.priv_data = (uintptr_t)xzalloc((args.priv_data_size));
> + if (!args.priv_data) {
> + ret = -ENOMEM;
> + goto exit;
> + }
> +
> + args.op = AMDGPU_CRIU_OP_CHECKPOINT;
> + ret = drmIoctl(fd, DRM_IOCTL_AMDGPU_CRIU_OP, &args);
> + if (ret) {
> + pr_perror("Failed to call dumper (process) ioctl");
> + goto exit;
> + }
> +
> + rd->priv_data.data = (void *)args.priv_data;
> + rd->priv_data.len = args.priv_data_size;
> +
> + for (int i = 0; i < args.num_bos; i++) {
> + struct drm_amdgpu_criu_bo_bucket bo_bucket = ((struct drm_amdgpu_criu_bo_bucket *)args.bos)[i];
> + uint32_t major, minor;
> + amdgpu_device_handle h_dev;
> + void *buffer = NULL;
> + char img_path[40];
> + FILE *bo_contents_fp = NULL;
> + DrmBoEntry *boinfo = rd->bo_entries[i];
> +
> + boinfo->addr = bo_bucket.addr;
> + boinfo->size = bo_bucket.size;
> + boinfo->offset = bo_bucket.offset;
> + boinfo->alloc_flags = bo_bucket.alloc_flags;
> + boinfo->preferred_domains = bo_bucket.preferred_domains;
> +
> + ret = amdgpu_device_initialize(fd, &major, &minor, &h_dev);
> +
> + snprintf(img_path, sizeof(img_path), IMG_DRM_PAGES_FILE, rd->pid, rd->drm_render_minor, i); //TODO: needs to be unique by process and by device, and recoverable by restore
> + bo_contents_fp = open_img_file(img_path, true, &image_size);
> +
> + posix_memalign(&buffer, sysconf(_SC_PAGE_SIZE), bo_bucket.size);
> +
> + ret = sdma_copy_bo(bo_bucket.dmabuf_fd, bo_bucket.size, bo_contents_fp, buffer, bo_bucket.size, h_dev, 0x1000,
> + SDMA_OP_VRAM_READ, false);
> +
> + boinfo->handle = get_gem_handle(h_dev, bo_bucket.dmabuf_fd);
> + boinfo->is_import = bo_bucket.is_import | shared_bo_has_exporter(boinfo->handle);
> +
> + if (bo_bucket.dmabuf_fd != KFD_INVALID_FD)
> + close(bo_bucket.dmabuf_fd);
> +
> + if (bo_contents_fp)
> + fclose(bo_contents_fp);
> +
> + ret = amdgpu_device_deinitialize(h_dev);
> + if (ret)
> + goto exit;
> + }
> + for (int i = 0; i < args.num_bos; i++) {
> + DrmBoEntry *boinfo = rd->bo_entries[i];
> +
> + ret = record_shared_bo(boinfo->handle, boinfo->is_import);
> + if (ret)
> + goto exit;
> + }
> +
> tp_node = sys_get_node_by_render_minor(&src_topology, minor);
> if (!tp_node) {
> pr_err("Failed to find a device with minor number = %d\n", minor);
> @@ -80,21 +324,129 @@ int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm)
> }
>
> /* Get the GPU_ID of the DRM device */
> - rd.gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id);
> - if (!rd.gpu_id) {
> - pr_err("Failed to find valid gpu_id for the device = %d\n", rd.gpu_id);
> + rd->gpu_id = maps_get_dest_gpu(&checkpoint_maps, tp_node->gpu_id);
> + if (!rd->gpu_id) {
> + pr_err("Failed to find valid gpu_id for the device = %d\n", rd->gpu_id);
> return -ENODEV;
> }
>
> - len = criu_render_node__get_packed_size(&rd);
> + len = criu_render_node__get_packed_size(rd);
> buf = xmalloc(len);
> if (!buf)
> return -ENOMEM;
>
> - criu_render_node__pack(&rd, buf);
> + criu_render_node__pack(rd, buf);
>
> snprintf(path, sizeof(path), IMG_DRM_FILE, id);
> ret = write_img_file(path, buf, len);
> +
> + exit:
> + xfree((void *)args.bos);
> + xfree((void *)args.priv_data);
> xfree(buf);
> + free_e(rd);
> + return ret;
> +}
> +
> +int amdgpu_plugin_drm_unpause_file(int fd) {
> + struct drm_amdgpu_criu_args args = {0};
> + int ret = 0;
> +
> + args.op = AMDGPU_CRIU_OP_UNPAUSE;
> + if (drmIoctl(fd, DRM_IOCTL_AMDGPU_CRIU_OP, &args) == -1) {
> + pr_perror("Failed to call unpause ioctl");
> + ret = -1;
> + goto exit;
> + }
> +
> + exit:
> return ret;
> }
> +
> +int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd)
> +{
> + struct drm_amdgpu_criu_args args = {0};
> + int ret = 0;
> + bool retry_needed = false;
> +
> + args.num_bos = rd->num_of_bos;
> + args.num_objs = rd->num_of_objects;
> + args.priv_data = (uint64_t)rd->priv_data.data;
> + args.priv_data_size = rd->priv_data.len;
> + args.bos = (uint64_t)xzalloc(sizeof(struct drm_amdgpu_criu_bo_bucket) * rd->num_of_bos);
> +
> + for (int i = 0; i < args.num_bos; i++) {
> + struct drm_amdgpu_criu_bo_bucket *bo_bucket = &((struct drm_amdgpu_criu_bo_bucket *)args.bos)[i];
> + DrmBoEntry *boinfo = rd->bo_entries[i];
> + int dmabuf_fd = -1;
> +
> + bo_bucket->addr = boinfo->addr;
> +
> + if (work_already_completed(boinfo->handle, rd->drm_render_minor)) {
> + bo_bucket->skip = 1;
> + } else if (boinfo->handle != -1) {
> + if (boinfo->is_import) {
> + dmabuf_fd = dmabuf_fd_for_handle(boinfo->handle);
> + if (dmabuf_fd == -1) {
> + bo_bucket->skip = 1;
> + retry_needed = true;
> + }
> + }
> + }
> +
> + bo_bucket->is_import = boinfo->is_import;
> +
> + bo_bucket->dmabuf_fd = dmabuf_fd;
> + bo_bucket->size = boinfo->size;
> + bo_bucket->offset = boinfo->offset;
> + bo_bucket->alloc_flags = boinfo->alloc_flags;
> + bo_bucket->preferred_domains = boinfo->preferred_domains;
> + }
> +
> + args.op = AMDGPU_CRIU_OP_RESTORE;
> + if (drmIoctl(fd, DRM_IOCTL_AMDGPU_CRIU_OP, &args) == -1) {
> + pr_perror("Failed to call restore ioctl");
> + ret = -1;
> + goto exit;
> + }
> +
> + for (int i = 0; i < args.num_bos; i++) {
> + struct drm_amdgpu_criu_bo_bucket *bo_bucket = &((struct drm_amdgpu_criu_bo_bucket *)args.bos)[i];
> + DrmBoEntry *boinfo = rd->bo_entries[i];
> +
> + if (!bo_bucket->skip && !work_already_completed(boinfo->handle, rd->drm_render_minor)) {
> + ret = record_completed_work(boinfo->handle, rd->drm_render_minor);
> + if (ret)
> + goto exit;
> + if (!boinfo->is_import) {
> + serve_out_dmabuf_fd(boinfo->handle, bo_bucket->dmabuf_fd);
> + }
> + }
> + }
> + ret = record_completed_work(-1, rd->drm_render_minor);
> + if (ret)
> + goto exit;
> +
> + if (args.num_bos > 0) {
> +
> + for (int i = 0; i < args.num_bos; i++) {
> + struct drm_amdgpu_criu_bo_bucket *bo_bucket = &((struct drm_amdgpu_criu_bo_bucket *)args.bos)[i];
> +
> + if (!bo_bucket->skip)
> + ret = save_vma_updates(bo_bucket->offset, bo_bucket->addr, bo_bucket->restored_offset, fd);
> + if (ret < 0)
> + goto exit;
> + }
> +
> + ret = restore_bo_contents_drm(rd->drm_render_minor, rd->pid, fd, args.num_bos, (struct drm_amdgpu_criu_bo_bucket *)args.bos);
> + if (ret)
> + goto exit;
> + }
> +
> +
> + exit:
> + if (ret < 0)
> + return ret;
> +
> + return retry_needed;
> +}
> diff --git a/plugins/amdgpu/amdgpu_plugin_drm.h b/plugins/amdgpu/amdgpu_plugin_drm.h
> index 6f0c1a9a6..be91912b0 100644
> --- a/plugins/amdgpu/amdgpu_plugin_drm.h
> +++ b/plugins/amdgpu/amdgpu_plugin_drm.h
> @@ -11,6 +11,7 @@
> #include "amdgpu_plugin_topology.h"
>
>
> +
> /**
> * Determines if VMA's of input file descriptor belong to amdgpu's
> * DRM device and are therefore supported
> @@ -24,5 +25,13 @@ int amdgpu_plugin_drm_handle_device_vma(int fd, const struct stat *drm);
> */
> int amdgpu_plugin_drm_dump_file(int fd, int id, struct stat *drm);
>
> +int amdgpu_plugin_drm_restore_file(int fd, CriuRenderNode *rd);
> +
> +int amdgpu_plugin_drm_unpause_file(int fd);
> +
> +int get_gem_handle(amdgpu_device_handle h_dev, int dmabuf_fd);
> +
> +int save_vma_updates(uint64_t offset, uint64_t addr, uint64_t restored_offset, int gpu_id);
> +
> #endif /* __AMDGPU_PLUGIN_DRM_H__ */
>
> diff --git a/plugins/amdgpu/amdgpu_plugin_util.c b/plugins/amdgpu/amdgpu_plugin_util.c
> index a165fc9cd..3a4b0652f 100644
> --- a/plugins/amdgpu/amdgpu_plugin_util.c
> +++ b/plugins/amdgpu/amdgpu_plugin_util.c
> @@ -39,7 +39,12 @@
> #include "amdgpu_plugin_topology.h"
>
> /* Tracks number of device files that need to be checkpointed */
> -static int dev_file_cnt = 0;
> +
> +
> +static LIST_HEAD(dumped_fds);
> +static LIST_HEAD(shared_bos);
> +static LIST_HEAD(shared_dmabuf_fds);
> +static LIST_HEAD(completed_work);
>
> /* Helper structures to encode device topology of SRC and DEST platforms */
> struct tp_system src_topology;
> @@ -49,37 +54,149 @@ struct tp_system dest_topology;
> struct device_maps checkpoint_maps;
> struct device_maps restore_maps;
>
> -bool checkpoint_is_complete()
> -{
> - return (dev_file_cnt == 0);
> +int record_dumped_fd(int fd, bool is_drm) {
> + int newfd = dup(fd);
> +
> + if (newfd < 0)
> + return newfd;
> + struct dumped_fd *st = malloc(sizeof(struct dumped_fd));
> + if (!st)
> + return -1;
> + st->fd = newfd;
> + st->is_drm = is_drm;
> + list_add(&st->l, &dumped_fds);
> +
> + return 0;
> }
>
> -void decrement_checkpoint_count()
> -{
> - dev_file_cnt--;
> +struct list_head *get_dumped_fds() {
> + return &dumped_fds;
> }
>
> -void init_gpu_count(struct tp_system *topo)
> -{
> - if (dev_file_cnt != 0)
> - return;
> +bool shared_bo_has_exporter(int handle) {
> + struct shared_bo *bo;
> +
> + if (handle == -1) {
> + return false;
> + }
>
> - /* We add ONE to include checkpointing of KFD device */
> - dev_file_cnt = 1 + topology_gpu_count(topo);
> + list_for_each_entry(bo, &shared_bos, l) {
> + if (bo->handle == handle) {
> + return bo->has_exporter;
> + }
> + }
> +
> + return false;
> }
>
> -int read_fp(FILE *fp, void *buf, const size_t buf_len)
> -{
> - size_t len_read;
> +int record_shared_bo(int handle, bool is_imported) {
> + struct shared_bo *bo;
>
> - len_read = fread(buf, 1, buf_len, fp);
> - if (len_read != buf_len) {
> - pr_err("Unable to read file (read:%ld buf_len:%ld)\n", len_read, buf_len);
> - return -EIO;
> + if (handle == -1)
> + return 0;
> +
> + list_for_each_entry(bo, &shared_bos, l) {
> + if (bo->handle == handle) {
> + return 0;
> + }
> }
> + bo = malloc(sizeof(struct shared_bo));
> + if (!bo)
> + return -1;
> + bo->handle = handle;
> + bo->has_exporter = !is_imported;
> + list_add(&bo->l, &shared_bos);
> +
> return 0;
> }
>
> +int record_shared_dmabuf_fd(int handle, int dmabuf_fd) {
> + struct shared_dmabuf *bo;
> +
> + bo = malloc(sizeof(struct shared_dmabuf));
> + if(!bo)
> + return -1;
> + bo->handle = handle;
> + bo->dmabuf_fd = dmabuf_fd;
> + list_add(&bo->l, &shared_dmabuf_fds);
> +
> + return 0;
> +}
> +
> +int dmabuf_fd_for_handle(int handle) {
> + struct shared_dmabuf *bo;
> +
> + list_for_each_entry(bo, &shared_dmabuf_fds, l) {
> + if (bo->handle == handle) {
> + return bo->dmabuf_fd;
> + }
> + }
> +
> + return -1;
> +}
> +
> +int record_completed_work(int handle, int id) {
> + struct restore_completed_work *work;
> +
> + work = malloc(sizeof(struct restore_completed_work));
> + if (!work)
> + return -1;
> + work->handle = handle;
> + work->id = id;
> + list_add(&work->l, &completed_work);
> +
> + return 0;
> +}
> +
> +bool work_already_completed(int handle, int id) {
> + struct restore_completed_work *work;
> +
> + list_for_each_entry(work, &completed_work, l) {
> + if (work->handle == handle && work->id == id) {
> + return true;
> + }
> + }
> +
> + return false;
> +}
> +
> +void clear_completed_work_and_dmabuf_fds() {
> + while (!list_empty(&shared_dmabuf_fds)) {
> + struct shared_dmabuf *st = list_first_entry(&shared_dmabuf_fds, struct shared_dmabuf, l);
> + list_del(&st->l);
> + close(st->dmabuf_fd);
> + free(st);
> + }
> +
> + while (!list_empty(&completed_work)) {
> + struct restore_completed_work *st = list_first_entry(&completed_work, struct restore_completed_work, l);
> + list_del(&st->l);
> + free(st);
> + }
> +}
> +
> +void clear_dumped_fds() {
> + while (!list_empty(&dumped_fds)) {
> + struct dumped_fd *st = list_first_entry(&dumped_fds, struct dumped_fd, l);
> + list_del(&st->l);
> + close(st->fd);
> + free(st);
> + }
> +}
> +
> +int read_fp(FILE *fp, void *buf, const size_t buf_len)
> +{
> + size_t len_read;
> +
> + len_read = fread(buf, 1, buf_len, fp);
> + if (len_read != buf_len) {
> + pr_err("Unable to read file (read:%ld buf_len:%ld)\n", len_read, buf_len);
> + return -EIO;
> +
> + }
> + return 0;
> +}
> +
> int write_fp(FILE *fp, const void *buf, const size_t buf_len)
> {
> size_t len_write;
> diff --git a/plugins/amdgpu/amdgpu_plugin_util.h b/plugins/amdgpu/amdgpu_plugin_util.h
> index aacca3a28..d64d18a02 100644
> --- a/plugins/amdgpu/amdgpu_plugin_util.h
> +++ b/plugins/amdgpu/amdgpu_plugin_util.h
> @@ -1,6 +1,8 @@
> #ifndef __AMDGPU_PLUGIN_UTIL_H__
> #define __AMDGPU_PLUGIN_UTIL_H__
>
> +#include <libdrm/amdgpu.h>
> +
> #ifndef _GNU_SOURCE
> #define _GNU_SOURCE 1
> #endif
> @@ -52,7 +54,7 @@
> #define IMG_DRM_FILE "amdgpu-renderD-%d.img"
>
> /* Name of file having serialized data of DRM device buffer objects (BOs) */
> -#define IMG_DRM_PAGES_FILE "amdgpu-drm-pages-%d-%04x.img"
> +#define IMG_DRM_PAGES_FILE "amdgpu-drm-pages-%d-%d-%04x.img"
>
> /* Helper macros to Checkpoint and Restore a ROCm file */
> #define HSAKMT_SHM_PATH "/dev/shm/hsakmt_shared_mem"
> @@ -73,6 +75,30 @@ enum sdma_op_type {
> SDMA_OP_VRAM_WRITE,
> };
>
> +struct dumped_fd {
> + struct list_head l;
> + int fd;
> + bool is_drm;
> +};
> +
> +struct shared_bo {
> + struct list_head l;
> + int handle;
> + bool has_exporter;
> +};
> +
> +struct shared_dmabuf {
> + struct list_head l;
> + int handle;
> + int dmabuf_fd;
> +};
> +
> +struct restore_completed_work {
> + struct list_head l;
> + int handle;
> + int id;
> +};
> +
> /* Helper structures to encode device topology of SRC and DEST platforms */
> extern struct tp_system src_topology;
> extern struct tp_system dest_topology;
> @@ -97,10 +123,25 @@ int read_file(const char *file_path, void *buf, const size_t buf_len);
> int write_img_file(char *path, const void *buf, const size_t buf_len);
> FILE *open_img_file(char *path, bool write, size_t *size);
>
> -bool checkpoint_is_complete();
> -void decrement_checkpoint_count();
> -void init_gpu_count(struct tp_system *topology);
> +int record_dumped_fd(int fd, bool is_drm);
> +struct list_head *get_dumped_fds();
> +void clear_dumped_fds();
> +
> +bool shared_bo_has_exporter(int handle);
> +int record_shared_bo(int handle, bool is_imported);
> +
> +int record_shared_dmabuf_fd(int handle, int dmabuf_fd);
> +int dmabuf_fd_for_handle(int handle);
> +
> +int record_completed_work(int handle, int id);
> +bool work_already_completed(int handle, int id);
> +
> +void clear_completed_work_and_dmabuf_fds();
>
> void print_kfd_bo_stat(int bo_cnt, struct kfd_criu_bo_bucket *bo_list);
>
> +int sdma_copy_bo(int shared_fd, uint64_t size, FILE *storage_fp,
> + void *buffer, size_t buffer_size, amdgpu_device_handle h_dev,
> + uint64_t max_copy_size, enum sdma_op_type type, bool do_not_free);
> +
> #endif /* __AMDGPU_PLUGIN_UTIL_H__ */
> diff --git a/plugins/amdgpu/criu-amdgpu.proto b/plugins/amdgpu/criu-amdgpu.proto
> index 078b67650..55b48a1c2 100644
> --- a/plugins/amdgpu/criu-amdgpu.proto
> +++ b/plugins/amdgpu/criu-amdgpu.proto
> @@ -46,6 +46,8 @@ message kfd_bo_entry {
> required uint64 offset = 3;
> required uint32 alloc_flags = 4;
> required uint32 gpu_id = 5;
> + required uint32 handle = 6;
> + required uint32 is_import = 7;
> }
>
> message criu_kfd {
> @@ -61,6 +63,22 @@ message criu_kfd {
> required bytes priv_data = 10;
> }
>
> +message drm_bo_entry {
> + required uint64 addr = 1;
> + required uint64 size = 2;
> + required uint64 offset = 3;
> + required uint64 alloc_flags = 4;
> + required uint32 preferred_domains = 5;
> + required uint32 handle = 6;
> + required uint32 is_import = 7;
> +}
> +
> message criu_render_node {
> required uint32 gpu_id = 1;
> + required uint32 pid = 2;
> + required uint32 drm_render_minor = 3;
> + required uint64 num_of_bos = 4;
> + repeated drm_bo_entry bo_entries = 5;
> + required uint32 num_of_objects = 6;
> + required bytes priv_data = 7;
> }
> diff --git a/plugins/amdgpu/kfd_ioctl.h b/plugins/amdgpu/kfd_ioctl.h
> index 1a3bcea95..8c3f3a518 100644
> --- a/plugins/amdgpu/kfd_ioctl.h
> +++ b/plugins/amdgpu/kfd_ioctl.h
> @@ -23,7 +23,7 @@
> #ifndef KFD_IOCTL_H_INCLUDED
> #define KFD_IOCTL_H_INCLUDED
>
> -#include <libdrm/drm.h>
> +#include <drm/drm.h>
> #include <linux/ioctl.h>
>
> /*
> @@ -34,84 +34,128 @@
> * - 1.6 - Query clear flags in SVM get_attr API
> * - 1.7 - Checkpoint Restore (CRIU) API
> * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
> + * - 1.9 - Add available memory ioctl
> + * - 1.10 - Add SMI profiler event log
> + * - 1.11 - Add unified memory for ctx save/restore area
> + * - 1.12 - Add DMA buf export ioctl
> + * - 1.13 - Add debugger API
> + * - 1.14 - Update kfd_event_data
> + * - 1.15 - Enable managing mappings in compute VMs with GEM_VA ioctl
> + * - 1.16 - Add contiguous VRAM allocation flag
> + * - 1.17 - Add SDMA queue creation with target SDMA engine ID
> */
> #define KFD_IOCTL_MAJOR_VERSION 1
> -#define KFD_IOCTL_MINOR_VERSION 8
> +#define KFD_IOCTL_MINOR_VERSION 17
>
> struct kfd_ioctl_get_version_args {
> - uint32_t major_version; /* from KFD */
> - uint32_t minor_version; /* from KFD */
> + __u32 major_version; /* from KFD */
> + __u32 minor_version; /* from KFD */
> };
>
> /* For kfd_ioctl_create_queue_args.queue_type. */
> -#define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0
> -#define KFD_IOC_QUEUE_TYPE_SDMA 0x1
> -#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2
> -#define KFD_IOC_QUEUE_TYPE_SDMA_XGMI 0x3
> +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0
> +#define KFD_IOC_QUEUE_TYPE_SDMA 0x1
> +#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2
> +#define KFD_IOC_QUEUE_TYPE_SDMA_XGMI 0x3
> +#define KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID 0x4
>
> -#define KFD_MAX_QUEUE_PERCENTAGE 100
> -#define KFD_MAX_QUEUE_PRIORITY 15
> +#define KFD_MAX_QUEUE_PERCENTAGE 100
> +#define KFD_MAX_QUEUE_PRIORITY 15
>
> struct kfd_ioctl_create_queue_args {
> - uint64_t ring_base_address; /* to KFD */
> - uint64_t write_pointer_address; /* from KFD */
> - uint64_t read_pointer_address; /* from KFD */
> - uint64_t doorbell_offset; /* from KFD */
> -
> - uint32_t ring_size; /* to KFD */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t queue_type; /* to KFD */
> - uint32_t queue_percentage; /* to KFD */
> - uint32_t queue_priority; /* to KFD */
> - uint32_t queue_id; /* from KFD */
> -
> - uint64_t eop_buffer_address; /* to KFD */
> - uint64_t eop_buffer_size; /* to KFD */
> - uint64_t ctx_save_restore_address; /* to KFD */
> - uint32_t ctx_save_restore_size; /* to KFD */
> - uint32_t ctl_stack_size; /* to KFD */
> + __u64 ring_base_address; /* to KFD */
> + __u64 write_pointer_address; /* from KFD */
> + __u64 read_pointer_address; /* from KFD */
> + __u64 doorbell_offset; /* from KFD */
> +
> + __u32 ring_size; /* to KFD */
> + __u32 gpu_id; /* to KFD */
> + __u32 queue_type; /* to KFD */
> + __u32 queue_percentage; /* to KFD */
> + __u32 queue_priority; /* to KFD */
> + __u32 queue_id; /* from KFD */
> +
> + __u64 eop_buffer_address; /* to KFD */
> + __u64 eop_buffer_size; /* to KFD */
> + __u64 ctx_save_restore_address; /* to KFD */
> + __u32 ctx_save_restore_size; /* to KFD */
> + __u32 ctl_stack_size; /* to KFD */
> + __u32 sdma_engine_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_destroy_queue_args {
> - uint32_t queue_id; /* to KFD */
> - uint32_t pad;
> + __u32 queue_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_update_queue_args {
> - uint64_t ring_base_address; /* to KFD */
> + __u64 ring_base_address; /* to KFD */
>
> - uint32_t queue_id; /* to KFD */
> - uint32_t ring_size; /* to KFD */
> - uint32_t queue_percentage; /* to KFD */
> - uint32_t queue_priority; /* to KFD */
> + __u32 queue_id; /* to KFD */
> + __u32 ring_size; /* to KFD */
> + __u32 queue_percentage; /* to KFD */
> + __u32 queue_priority; /* to KFD */
> };
>
> struct kfd_ioctl_set_cu_mask_args {
> - uint32_t queue_id; /* to KFD */
> - uint32_t num_cu_mask; /* to KFD */
> - uint64_t cu_mask_ptr; /* to KFD */
> + __u32 queue_id; /* to KFD */
> + __u32 num_cu_mask; /* to KFD */
> + __u64 cu_mask_ptr; /* to KFD */
> };
>
> struct kfd_ioctl_get_queue_wave_state_args {
> - uint64_t ctl_stack_address; /* to KFD */
> - uint32_t ctl_stack_used_size; /* from KFD */
> - uint32_t save_area_used_size; /* from KFD */
> - uint32_t queue_id; /* to KFD */
> - uint32_t pad;
> + __u64 ctl_stack_address; /* to KFD */
> + __u32 ctl_stack_used_size; /* from KFD */
> + __u32 save_area_used_size; /* from KFD */
> + __u32 queue_id; /* to KFD */
> + __u32 pad;
> +};
> +
> +struct kfd_ioctl_get_available_memory_args {
> + __u64 available; /* from KFD */
> + __u32 gpu_id; /* to KFD */
> + __u32 pad;
> +};
> +
> +struct kfd_dbg_device_info_entry {
> + __u64 exception_status;
> + __u64 lds_base;
> + __u64 lds_limit;
> + __u64 scratch_base;
> + __u64 scratch_limit;
> + __u64 gpuvm_base;
> + __u64 gpuvm_limit;
> + __u32 gpu_id;
> + __u32 location_id;
> + __u32 vendor_id;
> + __u32 device_id;
> + __u32 revision_id;
> + __u32 subsystem_vendor_id;
> + __u32 subsystem_device_id;
> + __u32 fw_version;
> + __u32 gfx_target_version;
> + __u32 simd_count;
> + __u32 max_waves_per_simd;
> + __u32 array_count;
> + __u32 simd_arrays_per_engine;
> + __u32 num_xcc;
> + __u32 capability;
> + __u32 debug_prop;
> };
>
> /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
> -#define KFD_IOC_CACHE_POLICY_COHERENT 0
> +#define KFD_IOC_CACHE_POLICY_COHERENT 0
> #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
>
> struct kfd_ioctl_set_memory_policy_args {
> - uint64_t alternate_aperture_base; /* to KFD */
> - uint64_t alternate_aperture_size; /* to KFD */
> + __u64 alternate_aperture_base; /* to KFD */
> + __u64 alternate_aperture_size; /* to KFD */
>
> - uint32_t gpu_id; /* to KFD */
> - uint32_t default_policy; /* to KFD */
> - uint32_t alternate_policy; /* to KFD */
> - uint32_t pad;
> + __u32 gpu_id; /* to KFD */
> + __u32 default_policy; /* to KFD */
> + __u32 alternate_policy; /* to KFD */
> + __u32 pad;
> };
>
> /*
> @@ -122,24 +166,24 @@ struct kfd_ioctl_set_memory_policy_args {
> */
>
> struct kfd_ioctl_get_clock_counters_args {
> - uint64_t gpu_clock_counter; /* from KFD */
> - uint64_t cpu_clock_counter; /* from KFD */
> - uint64_t system_clock_counter; /* from KFD */
> - uint64_t system_clock_freq; /* from KFD */
> + __u64 gpu_clock_counter; /* from KFD */
> + __u64 cpu_clock_counter; /* from KFD */
> + __u64 system_clock_counter; /* from KFD */
> + __u64 system_clock_freq; /* from KFD */
>
> - uint32_t gpu_id; /* to KFD */
> - uint32_t pad;
> + __u32 gpu_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_process_device_apertures {
> - uint64_t lds_base; /* from KFD */
> - uint64_t lds_limit; /* from KFD */
> - uint64_t scratch_base; /* from KFD */
> - uint64_t scratch_limit; /* from KFD */
> - uint64_t gpuvm_base; /* from KFD */
> - uint64_t gpuvm_limit; /* from KFD */
> - uint32_t gpu_id; /* from KFD */
> - uint32_t pad;
> + __u64 lds_base; /* from KFD */
> + __u64 lds_limit; /* from KFD */
> + __u64 scratch_base; /* from KFD */
> + __u64 scratch_limit; /* from KFD */
> + __u64 gpuvm_base; /* from KFD */
> + __u64 gpuvm_limit; /* from KFD */
> + __u32 gpu_id; /* from KFD */
> + __u32 pad;
> };
>
> /*
> @@ -149,122 +193,125 @@ struct kfd_process_device_apertures {
> */
> #define NUM_OF_SUPPORTED_GPUS 7
> struct kfd_ioctl_get_process_apertures_args {
> - struct kfd_process_device_apertures process_apertures[NUM_OF_SUPPORTED_GPUS]; /* from KFD */
> + struct kfd_process_device_apertures
> + process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */
>
> /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */
> - uint32_t num_of_nodes;
> - uint32_t pad;
> + __u32 num_of_nodes;
> + __u32 pad;
> };
>
> struct kfd_ioctl_get_process_apertures_new_args {
> /* User allocated. Pointer to struct kfd_process_device_apertures
> * filled in by Kernel
> */
> - uint64_t kfd_process_device_apertures_ptr;
> - /* to KFD - indicates amount of memory present in kfd_process_device_apertures_ptr
> + __u64 kfd_process_device_apertures_ptr;
> + /* to KFD - indicates amount of memory present in
> + * kfd_process_device_apertures_ptr
> * from KFD - Number of entries filled by KFD.
> */
> - uint32_t num_of_nodes;
> - uint32_t pad;
> + __u32 num_of_nodes;
> + __u32 pad;
> };
>
> -#define MAX_ALLOWED_NUM_POINTS 100
> -#define MAX_ALLOWED_AW_BUFF_SIZE 4096
> -#define MAX_ALLOWED_WAC_BUFF_SIZE 128
> +#define MAX_ALLOWED_NUM_POINTS 100
> +#define MAX_ALLOWED_AW_BUFF_SIZE 4096
> +#define MAX_ALLOWED_WAC_BUFF_SIZE 128
>
> struct kfd_ioctl_dbg_register_args {
> - uint32_t gpu_id; /* to KFD */
> - uint32_t pad;
> + __u32 gpu_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_dbg_unregister_args {
> - uint32_t gpu_id; /* to KFD */
> - uint32_t pad;
> + __u32 gpu_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_dbg_address_watch_args {
> - uint64_t content_ptr; /* a pointer to the actual content */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */
> + __u64 content_ptr; /* a pointer to the actual content */
> + __u32 gpu_id; /* to KFD */
> + __u32 buf_size_in_bytes; /*including gpu_id and buf_size */
> };
>
> struct kfd_ioctl_dbg_wave_control_args {
> - uint64_t content_ptr; /* a pointer to the actual content */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */
> + __u64 content_ptr; /* a pointer to the actual content */
> + __u32 gpu_id; /* to KFD */
> + __u32 buf_size_in_bytes; /*including gpu_id and buf_size */
> };
>
> -#define KFD_INVALID_FD 0xffffffff
> +#define KFD_INVALID_FD 0xffffffff
>
> /* Matching HSA_EVENTTYPE */
> -#define KFD_IOC_EVENT_SIGNAL 0
> -#define KFD_IOC_EVENT_NODECHANGE 1
> -#define KFD_IOC_EVENT_DEVICESTATECHANGE 2
> -#define KFD_IOC_EVENT_HW_EXCEPTION 3
> -#define KFD_IOC_EVENT_SYSTEM_EVENT 4
> -#define KFD_IOC_EVENT_DEBUG_EVENT 5
> -#define KFD_IOC_EVENT_PROFILE_EVENT 6
> -#define KFD_IOC_EVENT_QUEUE_EVENT 7
> -#define KFD_IOC_EVENT_MEMORY 8
> -
> -#define KFD_IOC_WAIT_RESULT_COMPLETE 0
> -#define KFD_IOC_WAIT_RESULT_TIMEOUT 1
> -#define KFD_IOC_WAIT_RESULT_FAIL 2
> -
> -#define KFD_SIGNAL_EVENT_LIMIT 4096
> +#define KFD_IOC_EVENT_SIGNAL 0
> +#define KFD_IOC_EVENT_NODECHANGE 1
> +#define KFD_IOC_EVENT_DEVICESTATECHANGE 2
> +#define KFD_IOC_EVENT_HW_EXCEPTION 3
> +#define KFD_IOC_EVENT_SYSTEM_EVENT 4
> +#define KFD_IOC_EVENT_DEBUG_EVENT 5
> +#define KFD_IOC_EVENT_PROFILE_EVENT 6
> +#define KFD_IOC_EVENT_QUEUE_EVENT 7
> +#define KFD_IOC_EVENT_MEMORY 8
> +
> +#define KFD_IOC_WAIT_RESULT_COMPLETE 0
> +#define KFD_IOC_WAIT_RESULT_TIMEOUT 1
> +#define KFD_IOC_WAIT_RESULT_FAIL 2
> +
> +#define KFD_SIGNAL_EVENT_LIMIT 4096
>
> /* For kfd_event_data.hw_exception_data.reset_type. */
> -#define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0
> -#define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1
> +#define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0
> +#define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1
>
> /* For kfd_event_data.hw_exception_data.reset_cause. */
> -#define KFD_HW_EXCEPTION_GPU_HANG 0
> -#define KFD_HW_EXCEPTION_ECC 1
> +#define KFD_HW_EXCEPTION_GPU_HANG 0
> +#define KFD_HW_EXCEPTION_ECC 1
>
> /* For kfd_hsa_memory_exception_data.ErrorType */
> -#define KFD_MEM_ERR_NO_RAS 0
> -#define KFD_MEM_ERR_SRAM_ECC 1
> -#define KFD_MEM_ERR_POISON_CONSUMED 2
> -#define KFD_MEM_ERR_GPU_HANG 3
> +#define KFD_MEM_ERR_NO_RAS 0
> +#define KFD_MEM_ERR_SRAM_ECC 1
> +#define KFD_MEM_ERR_POISON_CONSUMED 2
> +#define KFD_MEM_ERR_GPU_HANG 3
>
> struct kfd_ioctl_create_event_args {
> - uint64_t event_page_offset; /* from KFD */
> - uint32_t event_trigger_data; /* from KFD - signal events only */
> - uint32_t event_type; /* to KFD */
> - uint32_t auto_reset; /* to KFD */
> - uint32_t node_id; /* to KFD - only valid for certain event types */
> - uint32_t event_id; /* from KFD */
> - uint32_t event_slot_index; /* from KFD */
> + __u64 event_page_offset; /* from KFD */
> + __u32 event_trigger_data; /* from KFD - signal events only */
> + __u32 event_type; /* to KFD */
> + __u32 auto_reset; /* to KFD */
> + __u32 node_id; /* to KFD - only valid for certain
> + event types */
> + __u32 event_id; /* from KFD */
> + __u32 event_slot_index; /* from KFD */
> };
>
> struct kfd_ioctl_destroy_event_args {
> - uint32_t event_id; /* to KFD */
> - uint32_t pad;
> + __u32 event_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_set_event_args {
> - uint32_t event_id; /* to KFD */
> - uint32_t pad;
> + __u32 event_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_reset_event_args {
> - uint32_t event_id; /* to KFD */
> - uint32_t pad;
> + __u32 event_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_memory_exception_failure {
> - uint32_t NotPresent; /* Page not present or supervisor privilege */
> - uint32_t ReadOnly; /* Write access to a read-only page */
> - uint32_t NoExecute; /* Execute access to a page marked NX */
> - uint32_t imprecise; /* Can't determine the exact fault address */
> + __u32 NotPresent; /* Page not present or supervisor privilege */
> + __u32 ReadOnly; /* Write access to a read-only page */
> + __u32 NoExecute; /* Execute access to a page marked NX */
> + __u32 imprecise; /* Can't determine the exact fault address */
> };
>
> /* memory exception data */
> struct kfd_hsa_memory_exception_data {
> struct kfd_memory_exception_failure failure;
> - uint64_t va;
> - uint32_t gpu_id;
> - uint32_t ErrorType; /* 0 = no RAS error,
> + __u64 va;
> + __u32 gpu_id;
> + __u32 ErrorType; /* 0 = no RAS error,
> * 1 = ECC_SRAM,
> * 2 = Link_SYNFLOOD (poison),
> * 3 = GPU hang (not attributable to a specific cause),
> @@ -274,85 +321,98 @@ struct kfd_hsa_memory_exception_data {
>
> /* hw exception data */
> struct kfd_hsa_hw_exception_data {
> - uint32_t reset_type;
> - uint32_t reset_cause;
> - uint32_t memory_lost;
> - uint32_t gpu_id;
> + __u32 reset_type;
> + __u32 reset_cause;
> + __u32 memory_lost;
> + __u32 gpu_id;
> +};
> +
> +/* hsa signal event data */
> +struct kfd_hsa_signal_event_data {
> + __u64 last_event_age; /* to and from KFD */
> };
>
> /* Event data */
> struct kfd_event_data {
> union {
> + /* From KFD */
> struct kfd_hsa_memory_exception_data memory_exception_data;
> struct kfd_hsa_hw_exception_data hw_exception_data;
> - }; /* From KFD */
> - uint64_t kfd_event_data_ext; /* pointer to an extension structure for future exception types */
> - uint32_t event_id; /* to KFD */
> - uint32_t pad;
> + /* To and From KFD */
> + struct kfd_hsa_signal_event_data signal_event_data;
> + };
> + __u64 kfd_event_data_ext; /* pointer to an extension structure
> + for future exception types */
> + __u32 event_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_wait_events_args {
> - uint64_t events_ptr; /* pointed to struct kfd_event_data array, to KFD */
> - uint32_t num_events; /* to KFD */
> - uint32_t wait_for_all; /* to KFD */
> - uint32_t timeout; /* to KFD */
> - uint32_t wait_result; /* from KFD */
> + __u64 events_ptr; /* pointed to struct
> + kfd_event_data array, to KFD */
> + __u32 num_events; /* to KFD */
> + __u32 wait_for_all; /* to KFD */
> + __u32 timeout; /* to KFD */
> + __u32 wait_result; /* from KFD */
> };
>
> struct kfd_ioctl_set_scratch_backing_va_args {
> - uint64_t va_addr; /* to KFD */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t pad;
> + __u64 va_addr; /* to KFD */
> + __u32 gpu_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_get_tile_config_args {
> /* to KFD: pointer to tile array */
> - uint64_t tile_config_ptr;
> + __u64 tile_config_ptr;
> /* to KFD: pointer to macro tile array */
> - uint64_t macro_tile_config_ptr;
> + __u64 macro_tile_config_ptr;
> /* to KFD: array size allocated by user mode
> * from KFD: array size filled by kernel
> */
> - uint32_t num_tile_configs;
> + __u32 num_tile_configs;
> /* to KFD: array size allocated by user mode
> * from KFD: array size filled by kernel
> */
> - uint32_t num_macro_tile_configs;
> -
> - uint32_t gpu_id; /* to KFD */
> - uint32_t gb_addr_config; /* from KFD */
> - uint32_t num_banks; /* from KFD */
> - uint32_t num_ranks; /* from KFD */
> -
> - /* struct size can be extended later if needed without breaking ABI compatibility */
> + __u32 num_macro_tile_configs;
> +
> + __u32 gpu_id; /* to KFD */
> + __u32 gb_addr_config; /* from KFD */
> + __u32 num_banks; /* from KFD */
> + __u32 num_ranks; /* from KFD */
> + /* struct size can be extended later if needed
> + * without breaking ABI compatibility
> + */
> };
>
> struct kfd_ioctl_set_trap_handler_args {
> - uint64_t tba_addr; /* to KFD */
> - uint64_t tma_addr; /* to KFD */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t pad;
> + __u64 tba_addr; /* to KFD */
> + __u64 tma_addr; /* to KFD */
> + __u32 gpu_id; /* to KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_acquire_vm_args {
> - uint32_t drm_fd; /* to KFD */
> - uint32_t gpu_id; /* to KFD */
> + __u32 drm_fd; /* to KFD */
> + __u32 gpu_id; /* to KFD */
> };
>
> /* Allocation flags: memory types */
> -#define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4)
> /* Allocation flags: attributes/access options */
> -#define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26)
> -#define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24)
> +#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS (1 << 23)
>
> /* Allocate memory for later SVM (shared virtual memory) mapping.
> *
> @@ -367,12 +427,12 @@ struct kfd_ioctl_acquire_vm_args {
> * @flags: memory type and attributes. See KFD_IOC_ALLOC_MEM_FLAGS above
> */
> struct kfd_ioctl_alloc_memory_of_gpu_args {
> - uint64_t va_addr; /* to KFD */
> - uint64_t size; /* to KFD */
> - uint64_t handle; /* from KFD */
> - uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t flags;
> + __u64 va_addr; /* to KFD */
> + __u64 size; /* to KFD */
> + __u64 handle; /* from KFD */
> + __u64 mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */
> + __u32 gpu_id; /* to KFD */
> + __u32 flags;
> };
>
> /* Free memory allocated with kfd_ioctl_alloc_memory_of_gpu
> @@ -380,13 +440,13 @@ struct kfd_ioctl_alloc_memory_of_gpu_args {
> * @handle: memory handle returned by alloc
> */
> struct kfd_ioctl_free_memory_of_gpu_args {
> - uint64_t handle; /* to KFD */
> + __u64 handle; /* to KFD */
> };
>
> /* Map memory to one or more GPUs
> *
> * @handle: memory handle returned by alloc
> - * @device_ids_array_ptr: array of gpu_ids (uint32_t per device)
> + * @device_ids_array_ptr: array of gpu_ids (__u32 per device)
> * @n_devices: number of devices in the array
> * @n_success: number of devices mapped successfully
> *
> @@ -399,10 +459,10 @@ struct kfd_ioctl_free_memory_of_gpu_args {
> * n_devices.
> */
> struct kfd_ioctl_map_memory_to_gpu_args {
> - uint64_t handle; /* to KFD */
> - uint64_t device_ids_array_ptr; /* to KFD */
> - uint32_t n_devices; /* to KFD */
> - uint32_t n_success; /* to/from KFD */
> + __u64 handle; /* to KFD */
> + __u64 device_ids_array_ptr; /* to KFD */
> + __u32 n_devices; /* to KFD */
> + __u32 n_success; /* to/from KFD */
> };
>
> /* Unmap memory from one or more GPUs
> @@ -410,10 +470,10 @@ struct kfd_ioctl_map_memory_to_gpu_args {
> * same arguments as for mapping
> */
> struct kfd_ioctl_unmap_memory_from_gpu_args {
> - uint64_t handle; /* to KFD */
> - uint64_t device_ids_array_ptr; /* to KFD */
> - uint32_t n_devices; /* to KFD */
> - uint32_t n_success; /* to/from KFD */
> + __u64 handle; /* to KFD */
> + __u64 device_ids_array_ptr; /* to KFD */
> + __u32 n_devices; /* to KFD */
> + __u32 n_success; /* to/from KFD */
> };
>
> /* Allocate GWS for specific queue
> @@ -424,49 +484,167 @@ struct kfd_ioctl_unmap_memory_from_gpu_args {
> * only support contiguous GWS allocation
> */
> struct kfd_ioctl_alloc_queue_gws_args {
> - uint32_t queue_id; /* to KFD */
> - uint32_t num_gws; /* to KFD */
> - uint32_t first_gws; /* from KFD */
> - uint32_t pad;
> + __u32 queue_id; /* to KFD */
> + __u32 num_gws; /* to KFD */
> + __u32 first_gws; /* from KFD */
> + __u32 pad;
> };
>
> struct kfd_ioctl_get_dmabuf_info_args {
> - uint64_t size; /* from KFD */
> - uint64_t metadata_ptr; /* to KFD */
> - uint32_t metadata_size; /* to KFD (space allocated by user)
> - * from KFD (actual metadata size)
> - */
> - uint32_t gpu_id; /* from KFD */
> - uint32_t flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */
> - uint32_t dmabuf_fd; /* to KFD */
> + __u64 size; /* from KFD */
> + __u64 metadata_ptr; /* to KFD */
> + __u32 metadata_size; /* to KFD (space allocated by user)
> + * from KFD (actual metadata size)
> + */
> + __u32 gpu_id; /* from KFD */
> + __u32 flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */
> + __u32 dmabuf_fd; /* to KFD */
> };
>
> struct kfd_ioctl_import_dmabuf_args {
> - uint64_t va_addr; /* to KFD */
> - uint64_t handle; /* from KFD */
> - uint32_t gpu_id; /* to KFD */
> - uint32_t dmabuf_fd; /* to KFD */
> + __u64 va_addr; /* to KFD */
> + __u64 handle; /* from KFD */
> + __u32 gpu_id; /* to KFD */
> + __u32 dmabuf_fd; /* to KFD */
> +};
> +
> +struct kfd_ioctl_export_dmabuf_args {
> + __u64 handle; /* to KFD */
> + __u32 flags; /* to KFD */
> + __u32 dmabuf_fd; /* from KFD */
> };
>
> /*
> * KFD SMI(System Management Interface) events
> */
> enum kfd_smi_event {
> - KFD_SMI_EVENT_NONE = 0, /* not used */
> + KFD_SMI_EVENT_NONE = 0, /* not used */
> KFD_SMI_EVENT_VMFAULT = 1, /* event start counting at 1 */
> KFD_SMI_EVENT_THERMAL_THROTTLE = 2,
> KFD_SMI_EVENT_GPU_PRE_RESET = 3,
> KFD_SMI_EVENT_GPU_POST_RESET = 4,
> + KFD_SMI_EVENT_MIGRATE_START = 5,
> + KFD_SMI_EVENT_MIGRATE_END = 6,
> + KFD_SMI_EVENT_PAGE_FAULT_START = 7,
> + KFD_SMI_EVENT_PAGE_FAULT_END = 8,
> + KFD_SMI_EVENT_QUEUE_EVICTION = 9,
> + KFD_SMI_EVENT_QUEUE_RESTORE = 10,
> + KFD_SMI_EVENT_UNMAP_FROM_GPU = 11,
> +
> + /*
> + * max event number, as a flag bit to get events from all processes,
> + * this requires super user permission, otherwise will not be able to
> + * receive event from any process. Without this flag to receive events
> + * from same process.
> + */
> + KFD_SMI_EVENT_ALL_PROCESS = 64
> +};
> +
> +/* The reason of the page migration event */
> +enum KFD_MIGRATE_TRIGGERS {
> + KFD_MIGRATE_TRIGGER_PREFETCH, /* Prefetch to GPU VRAM or system memory */
> + KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, /* GPU page fault recover */
> + KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, /* CPU page fault recover */
> + KFD_MIGRATE_TRIGGER_TTM_EVICTION /* TTM eviction */
> +};
> +
> +/* The reason of user queue evition event */
> +enum KFD_QUEUE_EVICTION_TRIGGERS {
> + KFD_QUEUE_EVICTION_TRIGGER_SVM, /* SVM buffer migration */
> + KFD_QUEUE_EVICTION_TRIGGER_USERPTR, /* userptr movement */
> + KFD_QUEUE_EVICTION_TRIGGER_TTM, /* TTM move buffer */
> + KFD_QUEUE_EVICTION_TRIGGER_SUSPEND, /* GPU suspend */
> + KFD_QUEUE_EVICTION_CRIU_CHECKPOINT, /* CRIU checkpoint */
> + KFD_QUEUE_EVICTION_CRIU_RESTORE /* CRIU restore */
> +};
> +
> +/* The reason of unmap buffer from GPU event */
> +enum KFD_SVM_UNMAP_TRIGGERS {
> + KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY, /* MMU notifier CPU buffer movement */
> + KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE,/* MMU notifier page migration */
> + KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU /* Unmap to free the buffer */
> };
>
> -#define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i)-1))
> -#define KFD_SMI_EVENT_MSG_SIZE 96
> +#define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1))
> +#define KFD_SMI_EVENT_MSG_SIZE 96
>
> struct kfd_ioctl_smi_events_args {
> - uint32_t gpuid; /* to KFD */
> - uint32_t anon_fd; /* from KFD */
> + __u32 gpuid; /* to KFD */
> + __u32 anon_fd; /* from KFD */
> };
>
> +/*
> + * SVM event tracing via SMI system management interface
> + *
> + * Open event file descriptor
> + * use ioctl AMDKFD_IOC_SMI_EVENTS, pass in gpuid and return a anonymous file
> + * descriptor to receive SMI events.
> + * If calling with sudo permission, then file descriptor can be used to receive
> + * SVM events from all processes, otherwise, to only receive SVM events of same
> + * process.
> + *
> + * To enable the SVM event
> + * Write event file descriptor with KFD_SMI_EVENT_MASK_FROM_INDEX(event) bitmap
> + * mask to start record the event to the kfifo, use bitmap mask combination
> + * for multiple events. New event mask will overwrite the previous event mask.
> + * KFD_SMI_EVENT_MASK_FROM_INDEX(KFD_SMI_EVENT_ALL_PROCESS) bit requires sudo
> + * permisson to receive SVM events from all process.
> + *
> + * To receive the event
> + * Application can poll file descriptor to wait for the events, then read event
> + * from the file into a buffer. Each event is one line string message, starting
> + * with the event id, then the event specific information.
> + *
> + * To decode event information
> + * The following event format string macro can be used with sscanf to decode
> + * the specific event information.
> + * event triggers: the reason to generate the event, defined as enum for unmap,
> + * eviction and migrate events.
> + * node, from, to, prefetch_loc, preferred_loc: GPU ID, or 0 for system memory.
> + * addr: user mode address, in pages
> + * size: in pages
> + * pid: the process ID to generate the event
> + * ns: timestamp in nanosecond-resolution, starts at system boot time but
> + * stops during suspend
> + * migrate_update: GPU page fault is recovered by 'M' for migrate, 'U' for update
> + * rw: 'W' for write page fault, 'R' for read page fault
> + * rescheduled: 'R' if the queue restore failed and rescheduled to try again
> + * error_code: migrate failure error code, 0 if no error
> + */
> +#define KFD_EVENT_FMT_UPDATE_GPU_RESET(reset_seq_num, reset_cause)\
> + "%x %s\n", (reset_seq_num), (reset_cause)
> +
> +#define KFD_EVENT_FMT_THERMAL_THROTTLING(bitmask, counter)\
> + "%llx:%llx\n", (bitmask), (counter)
> +
> +#define KFD_EVENT_FMT_VMFAULT(pid, task_name)\
> + "%x:%s\n", (pid), (task_name)
> +
> +#define KFD_EVENT_FMT_PAGEFAULT_START(ns, pid, addr, node, rw)\
> + "%lld -%d @%lx(%x) %c\n", (ns), (pid), (addr), (node), (rw)
> +
> +#define KFD_EVENT_FMT_PAGEFAULT_END(ns, pid, addr, node, migrate_update)\
> + "%lld -%d @%lx(%x) %c\n", (ns), (pid), (addr), (node), (migrate_update)
> +
> +#define KFD_EVENT_FMT_MIGRATE_START(ns, pid, start, size, from, to, prefetch_loc,\
> + preferred_loc, migrate_trigger)\
> + "%lld -%d @%lx(%lx) %x->%x %x:%x %d\n", (ns), (pid), (start), (size),\
> + (from), (to), (prefetch_loc), (preferred_loc), (migrate_trigger)
> +
> +#define KFD_EVENT_FMT_MIGRATE_END(ns, pid, start, size, from, to, migrate_trigger, error_code) \
> + "%lld -%d @%lx(%lx) %x->%x %d %d\n", (ns), (pid), (start), (size),\
> + (from), (to), (migrate_trigger), (error_code)
> +
> +#define KFD_EVENT_FMT_QUEUE_EVICTION(ns, pid, node, evict_trigger)\
> + "%lld -%d %x %d\n", (ns), (pid), (node), (evict_trigger)
> +
> +#define KFD_EVENT_FMT_QUEUE_RESTORE(ns, pid, node, rescheduled)\
> + "%lld -%d %x %c\n", (ns), (pid), (node), (rescheduled)
> +
> +#define KFD_EVENT_FMT_UNMAP_FROM_GPU(ns, pid, addr, size, node, unmap_trigger)\
> + "%lld -%d @%lx(%lx) %x %d\n", (ns), (pid), (addr), (size),\
> + (node), (unmap_trigger)
> +
> /**************************************************************************************************
> * CRIU IOCTLs (Checkpoint Restore In Userspace)
> *
> @@ -503,40 +681,43 @@ enum kfd_criu_op {
> * @priv_data_size: [in/out] Size of priv_data in bytes
> * @num_devices: [in/out] Number of GPUs used by process. Size of @devices array.
> * @num_bos [in/out] Number of BOs used by process. Size of @bos array.
> - * @num_objects: [in/out] Number of objects used by process. Objects are opaque to user application.
> + * @num_objects: [in/out] Number of objects used by process. Objects are opaque to
> + * user application.
> * @pid: [in/out] PID of the process being checkpointed
> * @op [in] Type of operation (kfd_criu_op)
> *
> * Return: 0 on success, -errno on failure
> */
> struct kfd_ioctl_criu_args {
> - uint64_t devices; /* Used during ops: CHECKPOINT, RESTORE */
> - uint64_t bos; /* Used during ops: CHECKPOINT, RESTORE */
> - uint64_t priv_data; /* Used during ops: CHECKPOINT, RESTORE */
> - uint64_t priv_data_size; /* Used during ops: PROCESS_INFO, RESTORE */
> - uint32_t num_devices; /* Used during ops: PROCESS_INFO, RESTORE */
> - uint32_t num_bos; /* Used during ops: PROCESS_INFO, RESTORE */
> - uint32_t num_objects; /* Used during ops: PROCESS_INFO, RESTORE */
> - uint32_t pid; /* Used during ops: PROCESS_INFO, RESUME */
> - uint32_t op;
> + __u64 devices; /* Used during ops: CHECKPOINT, RESTORE */
> + __u64 bos; /* Used during ops: CHECKPOINT, RESTORE */
> + __u64 priv_data; /* Used during ops: CHECKPOINT, RESTORE */
> + __u64 priv_data_size; /* Used during ops: PROCESS_INFO, RESTORE */
> + __u32 num_devices; /* Used during ops: PROCESS_INFO, RESTORE */
> + __u32 num_bos; /* Used during ops: PROCESS_INFO, RESTORE */
> + __u32 num_objects; /* Used during ops: PROCESS_INFO, RESTORE */
> + __u32 pid; /* Used during ops: PROCESS_INFO, RESUME */
> + __u32 op;
> + __u8 is_retry: 1;
> };
>
> struct kfd_criu_device_bucket {
> - uint32_t user_gpu_id;
> - uint32_t actual_gpu_id;
> - uint32_t drm_fd;
> - uint32_t pad;
> + __u32 user_gpu_id;
> + __u32 actual_gpu_id;
> + __u32 drm_fd;
> + __u32 pad;
> };
>
> struct kfd_criu_bo_bucket {
> - uint64_t addr;
> - uint64_t size;
> - uint64_t offset;
> - uint64_t restored_offset; /* During restore, updated offset for BO */
> - uint32_t gpu_id; /* This is the user_gpu_id */
> - uint32_t alloc_flags;
> - uint32_t dmabuf_fd;
> - uint32_t pad;
> + __u64 addr;
> + __u64 size;
> + __u64 offset;
> + __u64 restored_offset; /* During restore, updated offset for BO */
> + __u32 gpu_id; /* This is the user_gpu_id */
> + __u32 alloc_flags;
> + __u32 dmabuf_fd;
> + __u8 is_import: 1;
> + __u8 skip: 1;
> };
>
> /* CRIU IOCTLs - END */
> @@ -552,15 +733,19 @@ enum kfd_mmio_remap {
> /* Guarantee host access to memory */
> #define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001
> /* Fine grained coherency between all devices with access */
> -#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002
> +#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002
> /* Use any GPU in same hive as preferred device */
> -#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004
> +#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004
> /* GPUs only read, allows replication */
> -#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008
> +#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008
> /* Allow execution on GPU */
> -#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010
> +#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010
> /* GPUs mostly read, may allow similar optimizations as RO, but writes fault */
> -#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020
> +#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020
> +/* Keep GPU memory mapping always valid as if XNACK is disable */
> +#define KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED 0x00000040
> +/* Fine grained coherency between all devices using device-scope atomics */
> +#define KFD_IOCTL_SVM_FLAG_EXT_COHERENT 0x00000080
>
> /**
> * kfd_ioctl_svm_op - SVM ioctl operations
> @@ -568,7 +753,10 @@ enum kfd_mmio_remap {
> * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes
> * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes
> */
> -enum kfd_ioctl_svm_op { KFD_IOCTL_SVM_OP_SET_ATTR, KFD_IOCTL_SVM_OP_GET_ATTR };
> +enum kfd_ioctl_svm_op {
> + KFD_IOCTL_SVM_OP_SET_ATTR,
> + KFD_IOCTL_SVM_OP_GET_ATTR
> +};
>
> /** kfd_ioctl_svm_location - Enum for preferred and prefetch locations
> *
> @@ -576,7 +764,10 @@ enum kfd_ioctl_svm_op { KFD_IOCTL_SVM_OP_SET_ATTR, KFD_IOCTL_SVM_OP_GET_ATTR };
> * Below definitions are used for system memory or for leaving the preferred
> * location unspecified.
> */
> -enum kfd_ioctl_svm_location { KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff };
> +enum kfd_ioctl_svm_location {
> + KFD_IOCTL_SVM_LOCATION_SYSMEM = 0,
> + KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff
> +};
>
> /**
> * kfd_ioctl_svm_attr_type - SVM attribute types
> @@ -616,8 +807,8 @@ enum kfd_ioctl_svm_attr_type {
> * @value: attribute value
> */
> struct kfd_ioctl_svm_attribute {
> - uint32_t type;
> - uint32_t value;
> + __u32 type;
> + __u32 value;
> };
>
> /**
> @@ -659,12 +850,12 @@ struct kfd_ioctl_svm_attribute {
> * attribute type to indicate the access for the specified GPU.
> */
> struct kfd_ioctl_svm_args {
> - uint64_t start_addr;
> - uint64_t size;
> - uint32_t op;
> - uint32_t nattr;
> + __u64 start_addr;
> + __u64 size;
> + __u32 op;
> + __u32 nattr;
> /* Variable length array of attributes */
> - struct kfd_ioctl_svm_attribute attrs[0];
> + struct kfd_ioctl_svm_attribute attrs[];
> };
>
> /**
> @@ -705,81 +896,773 @@ struct kfd_ioctl_set_xnack_mode_args {
> __s32 xnack_enabled;
> };
>
> -#define AMDKFD_IOCTL_BASE 'K'
> -#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr)
> -#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type)
> -#define AMDKFD_IOW(nr, type) _IOW(AMDKFD_IOCTL_BASE, nr, type)
> -#define AMDKFD_IOWR(nr, type) _IOWR(AMDKFD_IOCTL_BASE, nr, type)
> +/* Wave launch override modes */
> +enum kfd_dbg_trap_override_mode {
> + KFD_DBG_TRAP_OVERRIDE_OR = 0,
> + KFD_DBG_TRAP_OVERRIDE_REPLACE = 1
> +};
> +
> +/* Wave launch overrides */
> +enum kfd_dbg_trap_mask {
> + KFD_DBG_TRAP_MASK_FP_INVALID = 1,
> + KFD_DBG_TRAP_MASK_FP_INPUT_DENORMAL = 2,
> + KFD_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO = 4,
> + KFD_DBG_TRAP_MASK_FP_OVERFLOW = 8,
> + KFD_DBG_TRAP_MASK_FP_UNDERFLOW = 16,
> + KFD_DBG_TRAP_MASK_FP_INEXACT = 32,
> + KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO = 64,
> + KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH = 128,
> + KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION = 256,
> + KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START = (1 << 30),
> + KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END = (1 << 31)
> +};
> +
> +/* Wave launch modes */
> +enum kfd_dbg_trap_wave_launch_mode {
> + KFD_DBG_TRAP_WAVE_LAUNCH_MODE_NORMAL = 0,
> + KFD_DBG_TRAP_WAVE_LAUNCH_MODE_HALT = 1,
> + KFD_DBG_TRAP_WAVE_LAUNCH_MODE_DEBUG = 3
> +};
> +
> +/* Address watch modes */
> +enum kfd_dbg_trap_address_watch_mode {
> + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_READ = 0,
> + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_NONREAD = 1,
> + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_ATOMIC = 2,
> + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_ALL = 3
> +};
> +
> +/* Additional wave settings */
> +enum kfd_dbg_trap_flags {
> + KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP = 1,
> + KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP = 2,
> +};
> +
> +/* Trap exceptions */
> +enum kfd_dbg_trap_exception_code {
> + EC_NONE = 0,
> + /* per queue */
> + EC_QUEUE_WAVE_ABORT = 1,
> + EC_QUEUE_WAVE_TRAP = 2,
> + EC_QUEUE_WAVE_MATH_ERROR = 3,
> + EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION = 4,
> + EC_QUEUE_WAVE_MEMORY_VIOLATION = 5,
> + EC_QUEUE_WAVE_APERTURE_VIOLATION = 6,
> + EC_QUEUE_PACKET_DISPATCH_DIM_INVALID = 16,
> + EC_QUEUE_PACKET_DISPATCH_GROUP_SEGMENT_SIZE_INVALID = 17,
> + EC_QUEUE_PACKET_DISPATCH_CODE_INVALID = 18,
> + EC_QUEUE_PACKET_RESERVED = 19,
> + EC_QUEUE_PACKET_UNSUPPORTED = 20,
> + EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID = 21,
> + EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID = 22,
> + EC_QUEUE_PACKET_VENDOR_UNSUPPORTED = 23,
> + EC_QUEUE_PREEMPTION_ERROR = 30,
> + EC_QUEUE_NEW = 31,
> + /* per device */
> + EC_DEVICE_QUEUE_DELETE = 32,
> + EC_DEVICE_MEMORY_VIOLATION = 33,
> + EC_DEVICE_RAS_ERROR = 34,
> + EC_DEVICE_FATAL_HALT = 35,
> + EC_DEVICE_NEW = 36,
> + /* per process */
> + EC_PROCESS_RUNTIME = 48,
> + EC_PROCESS_DEVICE_REMOVE = 49,
> + EC_MAX
> +};
> +
> +/* Mask generated by ecode in kfd_dbg_trap_exception_code */
> +#define KFD_EC_MASK(ecode) (1ULL << (ecode - 1))
> +
> +/* Masks for exception code type checks below */
> +#define KFD_EC_MASK_QUEUE (KFD_EC_MASK(EC_QUEUE_WAVE_ABORT) | \
> + KFD_EC_MASK(EC_QUEUE_WAVE_TRAP) | \
> + KFD_EC_MASK(EC_QUEUE_WAVE_MATH_ERROR) | \
> + KFD_EC_MASK(EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION) | \
> + KFD_EC_MASK(EC_QUEUE_WAVE_MEMORY_VIOLATION) | \
> + KFD_EC_MASK(EC_QUEUE_WAVE_APERTURE_VIOLATION) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_DIM_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_GROUP_SEGMENT_SIZE_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_CODE_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_RESERVED) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_UNSUPPORTED) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_VENDOR_UNSUPPORTED) | \
> + KFD_EC_MASK(EC_QUEUE_PREEMPTION_ERROR) | \
> + KFD_EC_MASK(EC_QUEUE_NEW))
> +#define KFD_EC_MASK_DEVICE (KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE) | \
> + KFD_EC_MASK(EC_DEVICE_RAS_ERROR) | \
> + KFD_EC_MASK(EC_DEVICE_FATAL_HALT) | \
> + KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION) | \
> + KFD_EC_MASK(EC_DEVICE_NEW))
> +#define KFD_EC_MASK_PROCESS (KFD_EC_MASK(EC_PROCESS_RUNTIME) | \
> + KFD_EC_MASK(EC_PROCESS_DEVICE_REMOVE))
> +#define KFD_EC_MASK_PACKET (KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_DIM_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_GROUP_SEGMENT_SIZE_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_CODE_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_RESERVED) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_UNSUPPORTED) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID) | \
> + KFD_EC_MASK(EC_QUEUE_PACKET_VENDOR_UNSUPPORTED))
> +
> +/* Checks for exception code types for KFD search */
> +#define KFD_DBG_EC_IS_VALID(ecode) (ecode > EC_NONE && ecode < EC_MAX)
> +#define KFD_DBG_EC_TYPE_IS_QUEUE(ecode) \
> + (KFD_DBG_EC_IS_VALID(ecode) && !!(KFD_EC_MASK(ecode) & KFD_EC_MASK_QUEUE))
> +#define KFD_DBG_EC_TYPE_IS_DEVICE(ecode) \
> + (KFD_DBG_EC_IS_VALID(ecode) && !!(KFD_EC_MASK(ecode) & KFD_EC_MASK_DEVICE))
> +#define KFD_DBG_EC_TYPE_IS_PROCESS(ecode) \
> + (KFD_DBG_EC_IS_VALID(ecode) && !!(KFD_EC_MASK(ecode) & KFD_EC_MASK_PROCESS))
> +#define KFD_DBG_EC_TYPE_IS_PACKET(ecode) \
> + (KFD_DBG_EC_IS_VALID(ecode) && !!(KFD_EC_MASK(ecode) & KFD_EC_MASK_PACKET))
> +
> +
> +/* Runtime enable states */
> +enum kfd_dbg_runtime_state {
> + DEBUG_RUNTIME_STATE_DISABLED = 0,
> + DEBUG_RUNTIME_STATE_ENABLED = 1,
> + DEBUG_RUNTIME_STATE_ENABLED_BUSY = 2,
> + DEBUG_RUNTIME_STATE_ENABLED_ERROR = 3
> +};
> +
> +/* Runtime enable status */
> +struct kfd_runtime_info {
> + __u64 r_debug;
> + __u32 runtime_state;
> + __u32 ttmp_setup;
> +};
> +
> +/* Enable modes for runtime enable */
> +#define KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK 1
> +#define KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK 2
> +
> +/**
> + * kfd_ioctl_runtime_enable_args - Arguments for runtime enable
> + *
> + * Coordinates debug exception signalling and debug device enablement with runtime.
> + *
> + * @r_debug - pointer to user struct for sharing information between ROCr and the debuggger
> + * @mode_mask - mask to set mode
> + * KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK - enable runtime for debugging, otherwise disable
> + * KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK - enable trap temporary setup (ignore on disable)
> + * @capabilities_mask - mask to notify runtime on what KFD supports
> + *
> + * Return - 0 on SUCCESS.
> + * - EBUSY if runtime enable call already pending.
> + * - EEXIST if user queues already active prior to call.
> + * If process is debug enabled, runtime enable will enable debug devices and
> + * wait for debugger process to send runtime exception EC_PROCESS_RUNTIME
> + * to unblock - see kfd_ioctl_dbg_trap_args.
> + *
> + */
> +struct kfd_ioctl_runtime_enable_args {
> + __u64 r_debug;
> + __u32 mode_mask;
> + __u32 capabilities_mask;
> +};
> +
> +/* Queue information */
> +struct kfd_queue_snapshot_entry {
> + __u64 exception_status;
> + __u64 ring_base_address;
> + __u64 write_pointer_address;
> + __u64 read_pointer_address;
> + __u64 ctx_save_restore_address;
> + __u32 queue_id;
> + __u32 gpu_id;
> + __u32 ring_size;
> + __u32 queue_type;
> + __u32 ctx_save_restore_area_size;
> + __u32 reserved;
> +};
> +
> +/* Queue status return for suspend/resume */
> +#define KFD_DBG_QUEUE_ERROR_BIT 30
> +#define KFD_DBG_QUEUE_INVALID_BIT 31
> +#define KFD_DBG_QUEUE_ERROR_MASK (1 << KFD_DBG_QUEUE_ERROR_BIT)
> +#define KFD_DBG_QUEUE_INVALID_MASK (1 << KFD_DBG_QUEUE_INVALID_BIT)
> +
> +/* Context save area header information */
> +struct kfd_context_save_area_header {
> + struct {
> + __u32 control_stack_offset;
> + __u32 control_stack_size;
> + __u32 wave_state_offset;
> + __u32 wave_state_size;
> + } wave_state;
> + __u32 debug_offset;
> + __u32 debug_size;
> + __u64 err_payload_addr;
> + __u32 err_event_id;
> + __u32 reserved1;
> +};
> +
> +/*
> + * Debug operations
> + *
> + * For specifics on usage and return values, see documentation per operation
> + * below. Otherwise, generic error returns apply:
> + * - ESRCH if the process to debug does not exist.
> + *
> + * - EINVAL (with KFD_IOC_DBG_TRAP_ENABLE exempt) if operation
> + * KFD_IOC_DBG_TRAP_ENABLE has not succeeded prior.
> + * Also returns this error if GPU hardware scheduling is not supported.
> + *
> + * - EPERM (with KFD_IOC_DBG_TRAP_DISABLE exempt) if target process is not
> + * PTRACE_ATTACHED. KFD_IOC_DBG_TRAP_DISABLE is exempt to allow
> + * clean up of debug mode as long as process is debug enabled.
> + *
> + * - EACCES if any DBG_HW_OP (debug hardware operation) is requested when
> + * AMDKFD_IOC_RUNTIME_ENABLE has not succeeded prior.
> + *
> + * - ENODEV if any GPU does not support debugging on a DBG_HW_OP call.
> + *
> + * - Other errors may be returned when a DBG_HW_OP occurs while the GPU
> + * is in a fatal state.
> + *
> + */
> +enum kfd_dbg_trap_operations {
> + KFD_IOC_DBG_TRAP_ENABLE = 0,
> + KFD_IOC_DBG_TRAP_DISABLE = 1,
> + KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT = 2,
> + KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED = 3,
> + KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE = 4, /* DBG_HW_OP */
> + KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE = 5, /* DBG_HW_OP */
> + KFD_IOC_DBG_TRAP_SUSPEND_QUEUES = 6, /* DBG_HW_OP */
> + KFD_IOC_DBG_TRAP_RESUME_QUEUES = 7, /* DBG_HW_OP */
> + KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH = 8, /* DBG_HW_OP */
> + KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH = 9, /* DBG_HW_OP */
> + KFD_IOC_DBG_TRAP_SET_FLAGS = 10,
> + KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT = 11,
> + KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO = 12,
> + KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT = 13,
> + KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT = 14
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_enable_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_ENABLE.
> + *
> + * Enables debug session for target process. Call @op KFD_IOC_DBG_TRAP_DISABLE in
> + * kfd_ioctl_dbg_trap_args to disable debug session.
> + *
> + * @exception_mask (IN) - exceptions to raise to the debugger
> + * @rinfo_ptr (IN) - pointer to runtime info buffer (see kfd_runtime_info)
> + * @rinfo_size (IN/OUT) - size of runtime info buffer in bytes
> + * @dbg_fd (IN) - fd the KFD will nofify the debugger with of raised
> + * exceptions set in exception_mask.
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * Copies KFD saved kfd_runtime_info to @rinfo_ptr on enable.
> + * Size of kfd_runtime saved by the KFD returned to @rinfo_size.
> + * - EBADF if KFD cannot get a reference to dbg_fd.
> + * - EFAULT if KFD cannot copy runtime info to rinfo_ptr.
> + * - EINVAL if target process is already debug enabled.
> + *
> + */
> +struct kfd_ioctl_dbg_trap_enable_args {
> + __u64 exception_mask;
> + __u64 rinfo_ptr;
> + __u32 rinfo_size;
> + __u32 dbg_fd;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_send_runtime_event_args
> + *
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT.
> + * Raises exceptions to runtime.
> + *
> + * @exception_mask (IN) - exceptions to raise to runtime
> + * @gpu_id (IN) - target device id
> + * @queue_id (IN) - target queue id
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * - ENODEV if gpu_id not found.
> + * If exception_mask contains EC_PROCESS_RUNTIME, unblocks pending
> + * AMDKFD_IOC_RUNTIME_ENABLE call - see kfd_ioctl_runtime_enable_args.
> + * All other exceptions are raised to runtime through err_payload_addr.
> + * See kfd_context_save_area_header.
> + */
> +struct kfd_ioctl_dbg_trap_send_runtime_event_args {
> + __u64 exception_mask;
> + __u32 gpu_id;
> + __u32 queue_id;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_set_exceptions_enabled_args
> + *
> + * Arguments for KFD_IOC_SET_EXCEPTIONS_ENABLED
> + * Set new exceptions to be raised to the debugger.
> + *
> + * @exception_mask (IN) - new exceptions to raise the debugger
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + */
> +struct kfd_ioctl_dbg_trap_set_exceptions_enabled_args {
> + __u64 exception_mask;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_set_wave_launch_override_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE
> + * Enable HW exceptions to raise trap.
> + *
> + * @override_mode (IN) - see kfd_dbg_trap_override_mode
> + * @enable_mask (IN/OUT) - reference kfd_dbg_trap_mask.
> + * IN is the override modes requested to be enabled.
> + * OUT is referenced in Return below.
> + * @support_request_mask (IN/OUT) - reference kfd_dbg_trap_mask.
> + * IN is the override modes requested for support check.
> + * OUT is referenced in Return below.
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * Previous enablement is returned in @enable_mask.
> + * Actual override support is returned in @support_request_mask.
> + * - EINVAL if override mode is not supported.
> + * - EACCES if trap support requested is not actually supported.
> + * i.e. enable_mask (IN) is not a subset of support_request_mask (OUT).
> + * Otherwise it is considered a generic error (see kfd_dbg_trap_operations).
> + */
> +struct kfd_ioctl_dbg_trap_set_wave_launch_override_args {
> + __u32 override_mode;
> + __u32 enable_mask;
> + __u32 support_request_mask;
> + __u32 pad;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_set_wave_launch_mode_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE
> + * Set wave launch mode.
> + *
> + * @mode (IN) - see kfd_dbg_trap_wave_launch_mode
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + */
> +struct kfd_ioctl_dbg_trap_set_wave_launch_mode_args {
> + __u32 launch_mode;
> + __u32 pad;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_suspend_queues_ags
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_SUSPEND_QUEUES
> + * Suspend queues.
> + *
> + * @exception_mask (IN) - raised exceptions to clear
> + * @queue_array_ptr (IN) - pointer to array of queue ids (u32 per queue id)
> + * to suspend
> + * @num_queues (IN) - number of queues to suspend in @queue_array_ptr
> + * @grace_period (IN) - wave time allowance before preemption
> + * per 1K GPU clock cycle unit
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Destruction of a suspended queue is blocked until the queue is
> + * resumed. This allows the debugger to access queue information and
> + * the its context save area without running into a race condition on
> + * queue destruction.
> + * Automatically copies per queue context save area header information
> + * into the save area base
> + * (see kfd_queue_snapshot_entry and kfd_context_save_area_header).
> + *
> + * Return - Number of queues suspended on SUCCESS.
> + * . KFD_DBG_QUEUE_ERROR_MASK and KFD_DBG_QUEUE_INVALID_MASK masked
> + * for each queue id in @queue_array_ptr array reports unsuccessful
> + * suspend reason.
> + * KFD_DBG_QUEUE_ERROR_MASK = HW failure.
> + * KFD_DBG_QUEUE_INVALID_MASK = queue does not exist, is new or
> + * is being destroyed.
> + */
> +struct kfd_ioctl_dbg_trap_suspend_queues_args {
> + __u64 exception_mask;
> + __u64 queue_array_ptr;
> + __u32 num_queues;
> + __u32 grace_period;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_resume_queues_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_RESUME_QUEUES
> + * Resume queues.
> + *
> + * @queue_array_ptr (IN) - pointer to array of queue ids (u32 per queue id)
> + * to resume
> + * @num_queues (IN) - number of queues to resume in @queue_array_ptr
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - Number of queues resumed on SUCCESS.
> + * KFD_DBG_QUEUE_ERROR_MASK and KFD_DBG_QUEUE_INVALID_MASK mask
> + * for each queue id in @queue_array_ptr array reports unsuccessful
> + * resume reason.
> + * KFD_DBG_QUEUE_ERROR_MASK = HW failure.
> + * KFD_DBG_QUEUE_INVALID_MASK = queue does not exist.
> + */
> +struct kfd_ioctl_dbg_trap_resume_queues_args {
> + __u64 queue_array_ptr;
> + __u32 num_queues;
> + __u32 pad;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_set_node_address_watch_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH
> + * Sets address watch for device.
> + *
> + * @address (IN) - watch address to set
> + * @mode (IN) - see kfd_dbg_trap_address_watch_mode
> + * @mask (IN) - watch address mask
> + * @gpu_id (IN) - target gpu to set watch point
> + * @id (OUT) - watch id allocated
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * Allocated watch ID returned to @id.
> + * - ENODEV if gpu_id not found.
> + * - ENOMEM if watch IDs can be allocated
> + */
> +struct kfd_ioctl_dbg_trap_set_node_address_watch_args {
> + __u64 address;
> + __u32 mode;
> + __u32 mask;
> + __u32 gpu_id;
> + __u32 id;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_clear_node_address_watch_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH
> + * Clear address watch for device.
> + *
> + * @gpu_id (IN) - target device to clear watch point
> + * @id (IN) - allocated watch id to clear
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * - ENODEV if gpu_id not found.
> + * - EINVAL if watch ID has not been allocated.
> + */
> +struct kfd_ioctl_dbg_trap_clear_node_address_watch_args {
> + __u32 gpu_id;
> + __u32 id;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_set_flags_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_SET_FLAGS
> + * Sets flags for wave behaviour.
> + *
> + * @flags (IN/OUT) - IN = flags to enable, OUT = flags previously enabled
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * - EACCESS if any debug device does not allow flag options.
> + */
> +struct kfd_ioctl_dbg_trap_set_flags_args {
> + __u32 flags;
> + __u32 pad;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_query_debug_event_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT
> + *
> + * Find one or more raised exceptions. This function can return multiple
> + * exceptions from a single queue or a single device with one call. To find
> + * all raised exceptions, this function must be called repeatedly until it
> + * returns -EAGAIN. Returned exceptions can optionally be cleared by
> + * setting the corresponding bit in the @exception_mask input parameter.
> + * However, clearing an exception prevents retrieving further information
> + * about it with KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO.
> + *
> + * @exception_mask (IN/OUT) - exception to clear (IN) and raised (OUT)
> + * @gpu_id (OUT) - gpu id of exceptions raised
> + * @queue_id (OUT) - queue id of exceptions raised
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on raised exception found
> + * Raised exceptions found are returned in @exception mask
> + * with reported source id returned in @gpu_id or @queue_id.
> + * - EAGAIN if no raised exception has been found
> + */
> +struct kfd_ioctl_dbg_trap_query_debug_event_args {
> + __u64 exception_mask;
> + __u32 gpu_id;
> + __u32 queue_id;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_query_exception_info_args
> + *
> + * Arguments KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO
> + * Get additional info on raised exception.
> + *
> + * @info_ptr (IN) - pointer to exception info buffer to copy to
> + * @info_size (IN/OUT) - exception info buffer size (bytes)
> + * @source_id (IN) - target gpu or queue id
> + * @exception_code (IN) - target exception
> + * @clear_exception (IN) - clear raised @exception_code exception
> + * (0 = false, 1 = true)
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * If @exception_code is EC_DEVICE_MEMORY_VIOLATION, copy @info_size(OUT)
> + * bytes of memory exception data to @info_ptr.
> + * If @exception_code is EC_PROCESS_RUNTIME, copy saved
> + * kfd_runtime_info to @info_ptr.
> + * Actual required @info_ptr size (bytes) is returned in @info_size.
> + */
> +struct kfd_ioctl_dbg_trap_query_exception_info_args {
> + __u64 info_ptr;
> + __u32 info_size;
> + __u32 source_id;
> + __u32 exception_code;
> + __u32 clear_exception;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_get_queue_snapshot_args
> + *
> + * Arguments KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT
> + * Get queue information.
> + *
> + * @exception_mask (IN) - exceptions raised to clear
> + * @snapshot_buf_ptr (IN) - queue snapshot entry buffer (see kfd_queue_snapshot_entry)
> + * @num_queues (IN/OUT) - number of queue snapshot entries
> + * The debugger specifies the size of the array allocated in @num_queues.
> + * KFD returns the number of queues that actually existed. If this is
> + * larger than the size specified by the debugger, KFD will not overflow
> + * the array allocated by the debugger.
> + *
> + * @entry_size (IN/OUT) - size per entry in bytes
> + * The debugger specifies sizeof(struct kfd_queue_snapshot_entry) in
> + * @entry_size. KFD returns the number of bytes actually populated per
> + * entry. The debugger should use the KFD_IOCTL_MINOR_VERSION to determine,
> + * which fields in struct kfd_queue_snapshot_entry are valid. This allows
> + * growing the ABI in a backwards compatible manner.
> + * Note that entry_size(IN) should still be used to stride the snapshot buffer in the
> + * event that it's larger than actual kfd_queue_snapshot_entry.
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * Copies @num_queues(IN) queue snapshot entries of size @entry_size(IN)
> + * into @snapshot_buf_ptr if @num_queues(IN) > 0.
> + * Otherwise return @num_queues(OUT) queue snapshot entries that exist.
> + */
> +struct kfd_ioctl_dbg_trap_queue_snapshot_args {
> + __u64 exception_mask;
> + __u64 snapshot_buf_ptr;
> + __u32 num_queues;
> + __u32 entry_size;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_get_device_snapshot_args
> + *
> + * Arguments for KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT
> + * Get device information.
> + *
> + * @exception_mask (IN) - exceptions raised to clear
> + * @snapshot_buf_ptr (IN) - pointer to snapshot buffer (see kfd_dbg_device_info_entry)
> + * @num_devices (IN/OUT) - number of debug devices to snapshot
> + * The debugger specifies the size of the array allocated in @num_devices.
> + * KFD returns the number of devices that actually existed. If this is
> + * larger than the size specified by the debugger, KFD will not overflow
> + * the array allocated by the debugger.
> + *
> + * @entry_size (IN/OUT) - size per entry in bytes
> + * The debugger specifies sizeof(struct kfd_dbg_device_info_entry) in
> + * @entry_size. KFD returns the number of bytes actually populated. The
> + * debugger should use KFD_IOCTL_MINOR_VERSION to determine, which fields
> + * in struct kfd_dbg_device_info_entry are valid. This allows growing the
> + * ABI in a backwards compatible manner.
> + * Note that entry_size(IN) should still be used to stride the snapshot buffer in the
> + * event that it's larger than actual kfd_dbg_device_info_entry.
> + *
> + * Generic errors apply (see kfd_dbg_trap_operations).
> + * Return - 0 on SUCCESS.
> + * Copies @num_devices(IN) device snapshot entries of size @entry_size(IN)
> + * into @snapshot_buf_ptr if @num_devices(IN) > 0.
> + * Otherwise return @num_devices(OUT) queue snapshot entries that exist.
> + */
> +struct kfd_ioctl_dbg_trap_device_snapshot_args {
> + __u64 exception_mask;
> + __u64 snapshot_buf_ptr;
> + __u32 num_devices;
> + __u32 entry_size;
> +};
> +
> +/**
> + * kfd_ioctl_dbg_trap_args
> + *
> + * Arguments to debug target process.
> + *
> + * @pid - target process to debug
> + * @op - debug operation (see kfd_dbg_trap_operations)
> + *
> + * @op determines which union struct args to use.
> + * Refer to kern docs for each kfd_ioctl_dbg_trap_*_args struct.
> + */
> +struct kfd_ioctl_dbg_trap_args {
> + __u32 pid;
> + __u32 op;
> +
> + union {
> + struct kfd_ioctl_dbg_trap_enable_args enable;
> + struct kfd_ioctl_dbg_trap_send_runtime_event_args send_runtime_event;
> + struct kfd_ioctl_dbg_trap_set_exceptions_enabled_args set_exceptions_enabled;
> + struct kfd_ioctl_dbg_trap_set_wave_launch_override_args launch_override;
> + struct kfd_ioctl_dbg_trap_set_wave_launch_mode_args launch_mode;
> + struct kfd_ioctl_dbg_trap_suspend_queues_args suspend_queues;
> + struct kfd_ioctl_dbg_trap_resume_queues_args resume_queues;
> + struct kfd_ioctl_dbg_trap_set_node_address_watch_args set_node_address_watch;
> + struct kfd_ioctl_dbg_trap_clear_node_address_watch_args clear_node_address_watch;
> + struct kfd_ioctl_dbg_trap_set_flags_args set_flags;
> + struct kfd_ioctl_dbg_trap_query_debug_event_args query_debug_event;
> + struct kfd_ioctl_dbg_trap_query_exception_info_args query_exception_info;
> + struct kfd_ioctl_dbg_trap_queue_snapshot_args queue_snapshot;
> + struct kfd_ioctl_dbg_trap_device_snapshot_args device_snapshot;
> + };
> +};
> +
> +#define AMDKFD_IOCTL_BASE 'K'
> +#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr)
> +#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type)
> +#define AMDKFD_IOW(nr, type) _IOW(AMDKFD_IOCTL_BASE, nr, type)
> +#define AMDKFD_IOWR(nr, type) _IOWR(AMDKFD_IOCTL_BASE, nr, type)
> +
> +#define AMDKFD_IOC_GET_VERSION \
> + AMDKFD_IOR(0x01, struct kfd_ioctl_get_version_args)
> +
> +#define AMDKFD_IOC_CREATE_QUEUE \
> + AMDKFD_IOWR(0x02, struct kfd_ioctl_create_queue_args)
> +
> +#define AMDKFD_IOC_DESTROY_QUEUE \
> + AMDKFD_IOWR(0x03, struct kfd_ioctl_destroy_queue_args)
> +
> +#define AMDKFD_IOC_SET_MEMORY_POLICY \
> + AMDKFD_IOW(0x04, struct kfd_ioctl_set_memory_policy_args)
>
> -#define AMDKFD_IOC_GET_VERSION AMDKFD_IOR(0x01, struct kfd_ioctl_get_version_args)
> +#define AMDKFD_IOC_GET_CLOCK_COUNTERS \
> + AMDKFD_IOWR(0x05, struct kfd_ioctl_get_clock_counters_args)
>
> -#define AMDKFD_IOC_CREATE_QUEUE AMDKFD_IOWR(0x02, struct kfd_ioctl_create_queue_args)
> +#define AMDKFD_IOC_GET_PROCESS_APERTURES \
> + AMDKFD_IOR(0x06, struct kfd_ioctl_get_process_apertures_args)
>
> -#define AMDKFD_IOC_DESTROY_QUEUE AMDKFD_IOWR(0x03, struct kfd_ioctl_destroy_queue_args)
> +#define AMDKFD_IOC_UPDATE_QUEUE \
> + AMDKFD_IOW(0x07, struct kfd_ioctl_update_queue_args)
>
> -#define AMDKFD_IOC_SET_MEMORY_POLICY AMDKFD_IOW(0x04, struct kfd_ioctl_set_memory_policy_args)
> +#define AMDKFD_IOC_CREATE_EVENT \
> + AMDKFD_IOWR(0x08, struct kfd_ioctl_create_event_args)
>
> -#define AMDKFD_IOC_GET_CLOCK_COUNTERS AMDKFD_IOWR(0x05, struct kfd_ioctl_get_clock_counters_args)
> +#define AMDKFD_IOC_DESTROY_EVENT \
> + AMDKFD_IOW(0x09, struct kfd_ioctl_destroy_event_args)
>
> -#define AMDKFD_IOC_GET_PROCESS_APERTURES AMDKFD_IOR(0x06, struct kfd_ioctl_get_process_apertures_args)
> +#define AMDKFD_IOC_SET_EVENT \
> + AMDKFD_IOW(0x0A, struct kfd_ioctl_set_event_args)
>
> -#define AMDKFD_IOC_UPDATE_QUEUE AMDKFD_IOW(0x07, struct kfd_ioctl_update_queue_args)
> +#define AMDKFD_IOC_RESET_EVENT \
> + AMDKFD_IOW(0x0B, struct kfd_ioctl_reset_event_args)
>
> -#define AMDKFD_IOC_CREATE_EVENT AMDKFD_IOWR(0x08, struct kfd_ioctl_create_event_args)
> +#define AMDKFD_IOC_WAIT_EVENTS \
> + AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args)
>
> -#define AMDKFD_IOC_DESTROY_EVENT AMDKFD_IOW(0x09, struct kfd_ioctl_destroy_event_args)
> +#define AMDKFD_IOC_DBG_REGISTER_DEPRECATED \
> + AMDKFD_IOW(0x0D, struct kfd_ioctl_dbg_register_args)
>
> -#define AMDKFD_IOC_SET_EVENT AMDKFD_IOW(0x0A, struct kfd_ioctl_set_event_args)
> +#define AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED \
> + AMDKFD_IOW(0x0E, struct kfd_ioctl_dbg_unregister_args)
>
> -#define AMDKFD_IOC_RESET_EVENT AMDKFD_IOW(0x0B, struct kfd_ioctl_reset_event_args)
> +#define AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED \
> + AMDKFD_IOW(0x0F, struct kfd_ioctl_dbg_address_watch_args)
>
> -#define AMDKFD_IOC_WAIT_EVENTS AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args)
> +#define AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED \
> + AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args)
>
> -#define AMDKFD_IOC_DBG_REGISTER_DEPRECATED AMDKFD_IOW(0x0D, struct kfd_ioctl_dbg_register_args)
> +#define AMDKFD_IOC_SET_SCRATCH_BACKING_VA \
> + AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args)
>
> -#define AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED AMDKFD_IOW(0x0E, struct kfd_ioctl_dbg_unregister_args)
> +#define AMDKFD_IOC_GET_TILE_CONFIG \
> + AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args)
>
> -#define AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED AMDKFD_IOW(0x0F, struct kfd_ioctl_dbg_address_watch_args)
> +#define AMDKFD_IOC_SET_TRAP_HANDLER \
> + AMDKFD_IOW(0x13, struct kfd_ioctl_set_trap_handler_args)
>
> -#define AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args)
> +#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \
> + AMDKFD_IOWR(0x14, \
> + struct kfd_ioctl_get_process_apertures_new_args)
>
> -#define AMDKFD_IOC_SET_SCRATCH_BACKING_VA AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args)
> +#define AMDKFD_IOC_ACQUIRE_VM \
> + AMDKFD_IOW(0x15, struct kfd_ioctl_acquire_vm_args)
>
> -#define AMDKFD_IOC_GET_TILE_CONFIG AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args)
> +#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \
> + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args)
>
> -#define AMDKFD_IOC_SET_TRAP_HANDLER AMDKFD_IOW(0x13, struct kfd_ioctl_set_trap_handler_args)
> +#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \
> + AMDKFD_IOW(0x17, struct kfd_ioctl_free_memory_of_gpu_args)
>
> -#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW AMDKFD_IOWR(0x14, struct kfd_ioctl_get_process_apertures_new_args)
> +#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \
> + AMDKFD_IOWR(0x18, struct kfd_ioctl_map_memory_to_gpu_args)
>
> -#define AMDKFD_IOC_ACQUIRE_VM AMDKFD_IOW(0x15, struct kfd_ioctl_acquire_vm_args)
> +#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \
> + AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args)
>
> -#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args)
> +#define AMDKFD_IOC_SET_CU_MASK \
> + AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args)
>
> -#define AMDKFD_IOC_FREE_MEMORY_OF_GPU AMDKFD_IOW(0x17, struct kfd_ioctl_free_memory_of_gpu_args)
> +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \
> + AMDKFD_IOWR(0x1B, struct kfd_ioctl_get_queue_wave_state_args)
>
> -#define AMDKFD_IOC_MAP_MEMORY_TO_GPU AMDKFD_IOWR(0x18, struct kfd_ioctl_map_memory_to_gpu_args)
> +#define AMDKFD_IOC_GET_DMABUF_INFO \
> + AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_dmabuf_info_args)
>
> -#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args)
> +#define AMDKFD_IOC_IMPORT_DMABUF \
> + AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args)
>
> -#define AMDKFD_IOC_SET_CU_MASK AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args)
> +#define AMDKFD_IOC_ALLOC_QUEUE_GWS \
> + AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args)
>
> -#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE AMDKFD_IOWR(0x1B, struct kfd_ioctl_get_queue_wave_state_args)
> +#define AMDKFD_IOC_SMI_EVENTS \
> + AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)
>
> -#define AMDKFD_IOC_GET_DMABUF_INFO AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_dmabuf_info_args)
> +#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args)
>
> -#define AMDKFD_IOC_IMPORT_DMABUF AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args)
> +#define AMDKFD_IOC_SET_XNACK_MODE \
> + AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
>
> -#define AMDKFD_IOC_ALLOC_QUEUE_GWS AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args)
> +#define AMDKFD_IOC_CRIU_OP \
> + AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
>
> -#define AMDKFD_IOC_SMI_EVENTS AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)
> +#define AMDKFD_IOC_AVAILABLE_MEMORY \
> + AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args)
>
> -#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args)
> +#define AMDKFD_IOC_EXPORT_DMABUF \
> + AMDKFD_IOWR(0x24, struct kfd_ioctl_export_dmabuf_args)
>
> -#define AMDKFD_IOC_SET_XNACK_MODE AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
> +#define AMDKFD_IOC_RUNTIME_ENABLE \
> + AMDKFD_IOWR(0x25, struct kfd_ioctl_runtime_enable_args)
>
> -#define AMDKFD_IOC_CRIU_OP AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
> +#define AMDKFD_IOC_DBG_TRAP \
> + AMDKFD_IOWR(0x26, struct kfd_ioctl_dbg_trap_args)
>
> -#define AMDKFD_COMMAND_START 0x01
> -#define AMDKFD_COMMAND_END 0x23
> +#define AMDKFD_COMMAND_START 0x01
> +#define AMDKFD_COMMAND_END 0x27
>
> #endif
--
Best regards, Tikhomirov Pavel
Senior Software Developer, Virtuozzo.
More information about the CRIU
mailing list