Skip to content

Commit

Permalink
src/hmem_ze: map device id to physical device
Browse files Browse the repository at this point in the history 
The devices returned from zeDeviceGet map to each subdevice (tile),
not physical PCI devices. This means on a system with 2 physical
devices, each with 2 tiles, ZE will return 4 devices total.
This conflicts with the shm IPC implementation which uses device
fds on each physical device.

This commit creates a mapping between the devices returned from
zeDeviceGet and the physical devices they are on and uses that
mapping internally to get the correct device fd for the IPC
implementation.

This is done by looking at the device domain and PCI bus IDs
returned from the device properties (known to be at slots 6 and 8,
respectively) and matching it to the domain and bus IsD in the file
name opened for the device fd, which is in the format
domain:bus:dev.func

Signed-off-by: Alexia Ingerson <alexia.ingerson@intel.com>
  • Loading branch information
@aingerson @j-xiong
aingerson authored and j-xiong committed Jan 13, 2024
1 parent df3ba10 commit dccd54c
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 30 deletions.
6 changes: 3 additions & 3 deletions include/ofi_hmem.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,10 +214,10 @@ bool ze_hmem_is_addr_valid(const void *addr, uint64_t *device, uint64_t *flags);
int ze_hmem_get_handle(void *dev_buf, size_t size, void **handle);
int ze_hmem_open_handle(void **handle, size_t size, uint64_t device,
void **ipc_ptr);
int ze_hmem_get_shared_handle(int dev_fd, void *dev_buf, int *ze_fd,
int ze_hmem_get_shared_handle(uint64_t device, void *dev_buf, int *ze_fd,
void **handle);
int ze_hmem_open_shared_handle(int dev_fd, void **handle, int *ze_fd,
uint64_t device, void **ipc_ptr);
int ze_hmem_open_shared_handle(uint64_t device, int *peer_fds, void **handle,
int *ze_fd, void **ipc_ptr);
int ze_hmem_close_handle(void *ipc_ptr);
bool ze_hmem_p2p_enabled(void);
int ze_hmem_get_ipc_handle_size(size_t *size);
Expand Down
3 changes: 1 addition & 2 deletions prov/shm/src/smr_ep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,8 +323,7 @@ static int smr_format_ze_ipc(struct smr_ep *ep, int64_t id, struct smr_cmd *cmd,
if (ret)
return ret;

ret = ze_hmem_get_shared_handle(ep->sock_info->my_fds[device],
base, &pend->fd,
ret = ze_hmem_get_shared_handle(device, base, &pend->fd,
(void **) &cmd->msg.data.ipc_info.ipc_handle);
if (ret)
return ret;
Expand Down
10 changes: 4 additions & 6 deletions prov/shm/src/smr_progress.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,9 +536,8 @@ static struct smr_pend_entry *smr_progress_ipc(struct smr_cmd *cmd,
struct smr_region *peer_smr;
struct smr_resp *resp;
void *base, *ptr;
uint64_t ipc_device;
int64_t id;
int ret, fd, ipc_fd;
int ret, ipc_fd;
ssize_t hmem_copy_ret;
struct ofi_mr_entry *mr_entry;
struct smr_domain *domain;
Expand All @@ -553,11 +552,10 @@ static struct smr_pend_entry *smr_progress_ipc(struct smr_cmd *cmd,
//TODO disable IPC if more than 1 interface is initialized
if (cmd->msg.data.ipc_info.iface == FI_HMEM_ZE) {
id = cmd->msg.hdr.id;
ipc_device = cmd->msg.data.ipc_info.device;
fd = ep->sock_info->peers[id].device_fds[ipc_device];
ret = ze_hmem_open_shared_handle(fd,
ret = ze_hmem_open_shared_handle(cmd->msg.data.ipc_info.device,
ep->sock_info->peers[id].device_fds,
(void **) &cmd->msg.data.ipc_info.ipc_handle,
&ipc_fd, ipc_device, &base);
&ipc_fd, &base);
} else {
ret = ofi_ipc_cache_search(domain->ipc_cache,
cmd->msg.hdr.id,
Expand Down
57 changes: 38 additions & 19 deletions src/hmem_ze.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@

struct ofi_ze_dev_info {
ze_command_queue_handle_t cmd_queue;
int pci_device;
int ordinal;
int index;
ze_device_uuid_t uuid;
Expand All @@ -59,6 +60,7 @@ static ze_context_handle_t context;
static ze_device_handle_t *devices = NULL;
static struct ofi_ze_dev_info *dev_info = NULL;
static int num_devices = 0;
static int num_pci_devices = 0;
static int *dev_fds = NULL;
static bool p2p_enabled = false;
static bool host_reg_enabled = true;
Expand Down Expand Up @@ -387,7 +389,10 @@ static int ze_hmem_init_fds(void)
DIR *dir;
struct dirent *ent = NULL;
char dev_name[NAME_MAX];
int i = 0, ret;
int ret, i;
char *str;
uint16_t domain_id;
uint8_t pci_id;

dir = opendir(dev_dir);
if (dir == NULL)
Expand All @@ -403,26 +408,38 @@ static int ze_hmem_init_fds(void)
if (ret < 0 || ret >= NAME_MAX)
goto err;

dev_fds[i] = open(dev_name, O_RDWR);
if (dev_fds[i] == -1)
dev_fds[num_pci_devices] = open(dev_name, O_RDWR);
if (dev_fds[num_pci_devices] == -1)
goto err;
i++;
str = strtok(ent->d_name, "-");
str = strtok(NULL, ":");
domain_id = (uint16_t) strtol(str, NULL, 16);
str = strtok(NULL, ":");
pci_id = (uint8_t) strtol(str, NULL, 16);
for (i = 0; i < num_devices; i++) {
if (dev_info[i].uuid.id[8] == pci_id &&
(uint16_t) dev_info[i].uuid.id[6] == domain_id)
dev_info[i].pci_device = num_pci_devices;
}
num_pci_devices++;
}

(void) closedir(dir);
return FI_SUCCESS;

err:
(void) closedir(dir);
FI_WARN(&core_prov, FI_LOG_CORE,
"Failed open device %d\n", i);
"Failed open device %d\n", num_pci_devices);
return -FI_EIO;
}

int ze_hmem_get_shared_handle(int dev_fd, void *dev_buf, int *ze_fd,
int ze_hmem_get_shared_handle(uint64_t device, void *dev_buf, int *ze_fd,
void **handle)
{
struct drm_prime_handle open_fd = {0, 0, 0};
ze_ipc_mem_handle_t ze_handle;
int dev_fd = dev_fds[dev_info[device].pci_device];
int ret;

assert(dev_fd != -1);
Expand All @@ -443,11 +460,12 @@ int ze_hmem_get_shared_handle(int dev_fd, void *dev_buf, int *ze_fd,
return FI_SUCCESS;
}

int ze_hmem_open_shared_handle(int dev_fd, void **handle, int *ze_fd,
uint64_t device, void **ipc_ptr)
int ze_hmem_open_shared_handle(uint64_t device, int *peer_fds, void **handle,
int *ze_fd, void **ipc_ptr)
{
struct drm_prime_handle open_fd = {0, 0, 0};
ze_ipc_mem_handle_t ze_handle;
int dev_fd = peer_fds[dev_info[device].pci_device];
int ret;

open_fd.flags = DRM_CLOEXEC | DRM_RDWR;
Expand Down Expand Up @@ -478,13 +496,14 @@ static int ze_hmem_init_fds(void)
return FI_SUCCESS;
}

int ze_hmem_get_shared_handle(int dev_fd, void *dev_buf, int *ze_fd,
int ze_hmem_get_shared_handle(uint64_t device, void *dev_buf, int *ze_fd,
void **handle)
{
return -FI_ENOSYS;
}
int ze_hmem_open_shared_handle(int dev_fd, void **handle, int *ze_fd,
uint64_t device, void **ipc_ptr)

int ze_hmem_open_shared_handle(uint64_t device, int *peer_fds, void **handle,
int *ze_fd, void **ipc_ptr)
{
return -FI_ENOSYS;
}
Expand Down Expand Up @@ -839,10 +858,6 @@ int ze_hmem_init(void)
for (i = 0; i < count; dev_fds[i++] = -1)
;

ret = ze_hmem_init_fds();
if (ret)
goto err;

for (num_devices = 0; num_devices < count; num_devices++) {
dev_info[num_devices].cl_pool = NULL;
ze_ret = ofi_zeDeviceGetProperties(devices[num_devices],
Expand All @@ -867,6 +882,10 @@ int ze_hmem_init(void)
}
}

ret = ze_hmem_init_fds();
if (ret)
goto err;

p2p_enabled = p2p;
return FI_SUCCESS;

Expand Down Expand Up @@ -1116,7 +1135,7 @@ int ze_hmem_get_id(const void *ptr, uint64_t *id)

int *ze_hmem_get_dev_fds(int *nfds)
{
*nfds = num_devices;
*nfds = num_pci_devices;
return dev_fds;
}

Expand Down Expand Up @@ -1375,14 +1394,14 @@ int ze_hmem_open_handle(void **handle, size_t size, uint64_t device,
return -FI_ENOSYS;
}

int ze_hmem_get_shared_handle(int dev_fd, void *dev_buf, int *ze_fd,
int ze_hmem_get_shared_handle(uint64_t device, void *dev_buf, int *ze_fd,
void **handle)
{
return -FI_ENOSYS;
}

int ze_hmem_open_shared_handle(int dev_fd, void **handle, int *ze_fd,
uint64_t device, void **ipc_ptr)
int ze_hmem_open_shared_handle(uint64_t device, int *peer_fds, void **handle,
int *ze_fd, void **ipc_ptr)
{
return -FI_ENOSYS;
}
Expand Down

0 comments on commit dccd54c

Please sign in to comment.