Skip to content

Commit ccfa51a

Browse files
fdavid-amdrst0git
authored andcommitted
plugin/amdgpu: Support for checkpoint of dmabuf fds
amdgpu libraries that use dmabuf fd to share GPU memory between processes close the dmabuf fds immediately after using them. However, it is possible that checkpoint of a process catches one of the dmabuf fds open. In that case, the amdgpu plugin needs to handle it. The checkpoint of the dmabuf fd does require the device file it was exported from to have already been dumped To identify which device this dmabuf fd was exprted from, attempt to import it on each device, then record the dmabuf handle it imports as. This handle can be used to restore it. Signed-off-by: David Francis <David.Francis@amd.com>
1 parent 102c228 commit ccfa51a

File tree

8 files changed

+291
-19
lines changed

8 files changed

+291
-19
lines changed

plugins/amdgpu/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ endif
2727
criu-amdgpu.pb-c.c: criu-amdgpu.proto
2828
protoc --proto_path=. --c_out=. criu-amdgpu.proto
2929

30-
amdgpu_plugin.so: amdgpu_plugin.c amdgpu_plugin_drm.c amdgpu_plugin_topology.c amdgpu_plugin_util.c criu-amdgpu.pb-c.c amdgpu_socket_utils.c
30+
amdgpu_plugin.so: amdgpu_plugin.c amdgpu_plugin_drm.c amdgpu_plugin_dmabuf.c amdgpu_plugin_topology.c amdgpu_plugin_util.c criu-amdgpu.pb-c.c amdgpu_socket_utils.c
3131
$(CC) $(PLUGIN_CFLAGS) $(shell $(COMPEL) includes) $^ -o $@ $(PLUGIN_INCLUDE) $(PLUGIN_LDFLAGS) $(LIBDRM_INC)
3232

3333
amdgpu_plugin_clean:

plugins/amdgpu/amdgpu_plugin.c

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
#include "rst-malloc.h"
3939

4040
#include "common/list.h"
41+
#include "amdgpu_plugin_dmabuf.h"
4142
#include "amdgpu_plugin_drm.h"
4243
#include "amdgpu_plugin_util.h"
4344
#include "amdgpu_plugin_topology.h"
@@ -46,6 +47,7 @@
4647
#include "img-streamer.h"
4748
#include "image.h"
4849
#include "cr_options.h"
50+
#include "util.h"
4951

5052
struct vma_metadata {
5153
struct list_head list;
@@ -1064,6 +1066,9 @@ int amdgpu_unpause_processes(int pid)
10641066
}
10651067
}
10661068

1069+
if (post_dump_dmabuf_check() < 0)
1070+
ret = -1;
1071+
10671072
exit:
10681073
pr_info("Process unpaused %s (ret:%d)\n", ret ? "Failed" : "Ok", ret);
10691074
clear_dumped_fds();
@@ -1399,7 +1404,17 @@ int amdgpu_plugin_dump_file(int fd, int id)
13991404
return -1;
14001405
}
14011406

1402-
/* Check whether this plugin was called for kfd or render nodes */
1407+
/* Check whether this plugin was called for kfd, dmabuf or render nodes */
1408+
ret = get_dmabuf_info(fd, &st);
1409+
if (ret < 0) {
1410+
pr_perror("Failed to get dmabuf info");
1411+
return -1;
1412+
}
1413+
if (ret == 0) {
1414+
pr_info("Dumping dmabuf fd = %d\n", fd);
1415+
return amdgpu_plugin_dmabuf_dump(fd, id);
1416+
}
1417+
14031418
if (major(st.st_rdev) != major(st_kfd.st_rdev) || minor(st.st_rdev) != 0) {
14041419

14051420
/* This is RenderD dumper plugin, for now just save renderD
@@ -1415,7 +1430,7 @@ int amdgpu_plugin_dump_file(int fd, int id)
14151430
return ret;
14161431

14171432
/* Need to return success here so that criu can call plugins for renderD nodes */
1418-
return ret;
1433+
return try_dump_dmabuf_list();
14191434
}
14201435

14211436
pr_info("%s() called for fd = %d\n", __func__, major(st.st_rdev));
@@ -1537,7 +1552,6 @@ static int restore_devices(struct kfd_ioctl_criu_args *args, CriuKfd *e)
15371552
int ret = 0, bucket_index = 0;
15381553

15391554
pr_debug("Restoring %d devices\n", e->num_of_gpus);
1540-
15411555
args->num_devices = e->num_of_gpus;
15421556
device_buckets = xzalloc(sizeof(*device_buckets) * args->num_devices);
15431557
if (!device_buckets)
@@ -1821,12 +1835,21 @@ int amdgpu_plugin_restore_file(int id, bool *retry_needed)
18211835
* first as we assume restore_maps is already filled. Need to fix this later.
18221836
*/
18231837
snprintf(img_path, sizeof(img_path), IMG_DRM_FILE, id);
1824-
pr_info("Restoring RenderD %s\n", img_path);
18251838

18261839
img_fp = open_img_file(img_path, false, &img_size);
1827-
if (!img_fp)
1828-
return -EINVAL;
1829-
1840+
if (!img_fp) {
1841+
ret = amdgpu_plugin_dmabuf_restore(id);
1842+
if (ret == 1) {
1843+
/* This is a dmabuf fd, but the corresponding buffer object that was
1844+
* exported to make it has not yet been restored. Need to try again
1845+
* later when the buffer object exists, so it can be re-exported.
1846+
*/
1847+
*retry_needed = true;
1848+
return 0;
1849+
}
1850+
return ret;
1851+
}
1852+
pr_info("Restoring RenderD %s\n", img_path);
18301853
pr_debug("RenderD Image file size:%ld\n", img_size);
18311854
buf = xmalloc(img_size);
18321855
if (!buf) {
Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#include <errno.h>
2+
#include <fcntl.h>
3+
#include <stdlib.h>
4+
#include <stdint.h>
5+
#include <stdbool.h>
6+
#include <string.h>
7+
#include <stdio.h>
8+
#include <unistd.h>
9+
10+
#include <sys/stat.h>
11+
#include <sys/mman.h>
12+
#include <sys/types.h>
13+
#include <linux/limits.h>
14+
15+
#include "common/list.h"
16+
#include "criu-amdgpu.pb-c.h"
17+
18+
#include "xmalloc.h"
19+
#include "criu-log.h"
20+
#include "amdgpu_plugin_drm.h"
21+
#include "amdgpu_plugin_util.h"
22+
#include "amdgpu_plugin_dmabuf.h"
23+
#include "fdstore.h"
24+
25+
#include "util.h"
26+
#include "common/scm.h"
27+
28+
struct dmabuf {
29+
int id;
30+
int dmabuf_fd;
31+
struct list_head node;
32+
};
33+
34+
static LIST_HEAD(dmabuf_list);
35+
36+
/* Return < 0 for error, > 0 for "not a dmabuf" and 0 "is a dmabuf" */
37+
int get_dmabuf_info(int fd, struct stat *st)
38+
{
39+
char path[PATH_MAX];
40+
41+
if (read_fd_link(fd, path, sizeof(path)) < 0)
42+
return -1;
43+
44+
if (strncmp(path, DMABUF_LINK, strlen(DMABUF_LINK)) != 0)
45+
return 1;
46+
47+
return 0;
48+
}
49+
50+
int __amdgpu_plugin_dmabuf_dump(int dmabuf_fd, int id)
51+
{
52+
int ret = 0;
53+
char path[PATH_MAX];
54+
size_t len = 0;
55+
unsigned char *buf = NULL;
56+
int gem_handle;
57+
58+
gem_handle = handle_for_shared_bo_fd(dmabuf_fd);
59+
if (gem_handle < 0) {
60+
pr_err("Failed to get handle for dmabuf_fd = %d\n", dmabuf_fd);
61+
return -EAGAIN; /* Retry needed */
62+
}
63+
64+
CriuDmabufNode *node = xmalloc(sizeof(*node));
65+
if (!node) {
66+
pr_err("Failed to allocate memory for dmabuf node\n");
67+
return -ENOMEM;
68+
}
69+
criu_dmabuf_node__init(node);
70+
71+
node->gem_handle = gem_handle;
72+
73+
if (node->gem_handle < 0) {
74+
pr_err("Failed to get handle for dmabuf_fd\n");
75+
xfree(node);
76+
return -EINVAL;
77+
}
78+
79+
/* Serialize metadata to a file */
80+
snprintf(path, sizeof(path), IMG_DMABUF_FILE, id);
81+
len = criu_dmabuf_node__get_packed_size(node);
82+
buf = xmalloc(len);
83+
if (!buf) {
84+
pr_err("Failed to allocate buffer for dmabuf metadata\n");
85+
xfree(node);
86+
return -ENOMEM;
87+
}
88+
criu_dmabuf_node__pack(node, buf);
89+
ret = write_img_file(path, buf, len);
90+
91+
xfree(buf);
92+
xfree(node);
93+
return ret;
94+
}
95+
96+
int amdgpu_plugin_dmabuf_restore(int id)
97+
{
98+
char path[PATH_MAX];
99+
size_t img_size;
100+
FILE *img_fp = NULL;
101+
int ret = 0;
102+
CriuDmabufNode *rd = NULL;
103+
unsigned char *buf = NULL;
104+
int fd_id;
105+
106+
snprintf(path, sizeof(path), IMG_DMABUF_FILE, id);
107+
108+
/* Read serialized metadata */
109+
img_fp = open_img_file(path, false, &img_size);
110+
if (!img_fp) {
111+
pr_err("Failed to open dmabuf metadata file: %s\n", path);
112+
return -EINVAL;
113+
}
114+
115+
pr_debug("dmabuf Image file size:%ld\n", img_size);
116+
buf = xmalloc(img_size);
117+
if (!buf) {
118+
pr_perror("Failed to allocate memory");
119+
return -ENOMEM;
120+
}
121+
122+
ret = read_fp(img_fp, buf, img_size);
123+
if (ret) {
124+
pr_perror("Unable to read from %s", path);
125+
xfree(buf);
126+
return ret;
127+
}
128+
129+
rd = criu_dmabuf_node__unpack(NULL, img_size, buf);
130+
if (rd == NULL) {
131+
pr_perror("Unable to parse the dmabuf message %d", id);
132+
xfree(buf);
133+
fclose(img_fp);
134+
return -1;
135+
}
136+
fclose(img_fp);
137+
138+
/* Match GEM handle with shared_dmabuf list */
139+
fd_id = amdgpu_id_for_handle(rd->gem_handle);
140+
if (fd_id == -1) {
141+
pr_err("Failed to find dmabuf_fd for GEM handle = %d\n", rd->gem_handle);
142+
return 1;
143+
}
144+
145+
int dmabuf_fd = fdstore_get(fd_id);
146+
if (dmabuf_fd == -1) {
147+
pr_err("Failed to find dmabuf_fd for GEM handle = %d\n", rd->gem_handle);
148+
return 1; /* Retry needed */
149+
}
150+
151+
pr_info("Restored dmabuf_fd = %d for GEM handle = %d\n", dmabuf_fd, rd->gem_handle);
152+
ret = dmabuf_fd;
153+
154+
pr_info("Successfully restored dmabuf_fd %d\n", dmabuf_fd);
155+
criu_dmabuf_node__free_unpacked(rd, NULL);
156+
xfree(buf);
157+
return ret;
158+
}
159+
160+
int amdgpu_plugin_dmabuf_dump(int dmabuf_fd, int id)
161+
{
162+
int ret;
163+
164+
ret = __amdgpu_plugin_dmabuf_dump(dmabuf_fd, id);
165+
if (ret == -EAGAIN) {
166+
struct dmabuf *b = xmalloc(sizeof(*b));
167+
b->id = id;
168+
b->dmabuf_fd = dmabuf_fd;
169+
list_add(&b->node, &dmabuf_list);
170+
return 0;
171+
}
172+
return ret;
173+
}
174+
175+
int try_dump_dmabuf_list()
176+
{
177+
struct dmabuf *b, *t;
178+
list_for_each_entry_safe(b, t, &dmabuf_list, node) {
179+
int ret = __amdgpu_plugin_dmabuf_dump(b->dmabuf_fd, b->id);
180+
if (ret == -EAGAIN)
181+
continue;
182+
if (ret)
183+
return ret;
184+
list_del(&b->node);
185+
xfree(b);
186+
}
187+
return 0;
188+
}
189+
190+
int post_dump_dmabuf_check()
191+
{
192+
if (!list_empty(&dmabuf_list)) {
193+
pr_err("Not all dma buffers have been dumped\n");
194+
return -1;
195+
}
196+
return 0;
197+
}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#ifndef __AMDGPU_PLUGIN_DMABUF_H__
2+
#define __AMDGPU_PLUGIN_DMABUF_H__
3+
4+
#include "amdgpu_plugin_util.h"
5+
#include "criu-amdgpu.pb-c.h"
6+
7+
int amdgpu_plugin_dmabuf_dump(int fd, int id);
8+
int amdgpu_plugin_dmabuf_restore(int id);
9+
10+
int try_dump_dmabuf_list();
11+
int post_dump_dmabuf_check();
12+
13+
int get_dmabuf_info(int fd, struct stat *st);
14+
15+
#endif /* __AMDGPU_PLUGIN_DMABUF_H__ */

plugins/amdgpu/amdgpu_plugin_drm.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ int get_gem_handle(amdgpu_device_handle h_dev, int dmabuf_fd)
4747
return -1;
4848
}
4949

50-
drmPrimeFDToHandle(fd, dmabuf_fd, &handle);
50+
if (drmPrimeFDToHandle(fd, dmabuf_fd, &handle))
51+
return -1;
5152

5253
return handle;
5354
}

plugins/amdgpu/amdgpu_plugin_util.c

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "amdgpu_drm.h"
3838
#include "amdgpu_plugin_util.h"
3939
#include "amdgpu_plugin_topology.h"
40+
#include "amdgpu_plugin_drm.h"
4041

4142
static LIST_HEAD(dumped_fds);
4243
static LIST_HEAD(shared_bos);
@@ -109,6 +110,42 @@ int record_shared_bo(int handle, bool is_imported)
109110
return 0;
110111
}
111112

113+
int handle_for_shared_bo_fd(int fd)
114+
{
115+
struct dumped_fd *df;
116+
int trial_handle;
117+
amdgpu_device_handle h_dev;
118+
uint32_t major, minor;
119+
struct shared_bo *bo;
120+
121+
list_for_each_entry(df, &dumped_fds, l) {
122+
/* see if the gem handle for fd using the hdev for df->fd is the
123+
same as bo->handle. */
124+
125+
if (!df->is_drm) {
126+
continue;
127+
}
128+
129+
if (amdgpu_device_initialize(df->fd, &major, &minor, &h_dev)) {
130+
pr_err("Failed to initialize amdgpu device\n");
131+
continue;
132+
}
133+
134+
trial_handle = get_gem_handle(h_dev, fd);
135+
if (trial_handle < 0)
136+
continue;
137+
138+
list_for_each_entry(bo, &shared_bos, l) {
139+
if (bo->handle == trial_handle)
140+
return trial_handle;
141+
}
142+
143+
amdgpu_device_deinitialize(h_dev);
144+
}
145+
146+
return -1;
147+
}
148+
112149
int record_completed_work(int handle, int id)
113150
{
114151
struct restore_completed_work *work;
@@ -138,13 +175,6 @@ bool work_already_completed(int handle, int id)
138175

139176
void clear_restore_state()
140177
{
141-
while (!list_empty(&shared_dmabuf_fds)) {
142-
struct shared_dmabuf *st = list_first_entry(&shared_dmabuf_fds, struct shared_dmabuf, l);
143-
list_del(&st->l);
144-
close(st->dmabuf_fd);
145-
free(st);
146-
}
147-
148178
while (!list_empty(&completed_work)) {
149179
struct restore_completed_work *st = list_first_entry(&completed_work, struct restore_completed_work, l);
150180
list_del(&st->l);

0 commit comments

Comments
 (0)