libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1586090:core:core:ze_hmem_dl_init():422 Failed to dlopen libze_loader.so libfabric:1586090:core:core:ofi_hmem_init():214 Failed to initialize hmem iface FI_HMEM_ZE: No data available libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1586090:core:mr:ofi_default_cache_size():78 default cache size=2518031923 libfabric:1601323:core:core:ze_hmem_dl_init():422 Failed to dlopen libze_loader.so libfabric:1601323:core:core:ofi_hmem_init():214 Failed to initialize hmem iface FI_HMEM_ZE: No data available libfabric:1601323:core:mr:ofi_default_cache_size():78 default cache size=2518031923 libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ZE not supported libfabric:1586090:core:mr:ofi_default_cache_size():78 default cache size=2518031923 libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ZE not supported libfabric:1601323:core:mr:ofi_default_cache_size():78 default cache size=2518031923 libfabric:1586090:verbs:fabric:verbs_devs_print():883 list of verbs devices found for FI_EP_MSG: libfabric:1586090:verbs:fabric:verbs_devs_print():887 #1 hfi1_0 - IPoIB addresses: libfabric:1586090:verbs:fabric:verbs_devs_print():897 10.10.8.57 libfabric:1586090:verbs:fabric:verbs_devs_print():897 fe80::211:7501:178:bd16 libfabric:1586090:verbs:fabric:vrb_get_device_attrs():618 device hfi1_0: first found active port is 1 libfabric:1586090:verbs:fabric:vrb_get_device_attrs():563 XRC support unavailable in device: hfi1_0 libfabric:1586090:verbs:fabric:vrb_get_device_attrs():618 device hfi1_0: first found active port is 1 libfabric:1586090:core:core:ofi_register_provider():474 registering provider: verbs (113.20) libfabric:1586090:core:core:ofi_register_provider():474 registering provider: tcp (113.20) libfabric:1586090:core:core:ofi_register_provider():474 registering provider: sockets (113.20) libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ZE not supported libfabric:1586090:core:core:ofi_register_provider():474 registering provider: shm (113.20) libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1586090:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1586090:core:core:ze_hmem_dl_init():422 Failed to dlopen libze_loader.so libfabric:1586090:core:core:ofi_hmem_init():214 Failed to initialize hmem iface FI_HMEM_ZE: No data available libfabric:1586090:core:core:ofi_register_provider():474 registering provider: ofi_rxm (113.20) libfabric:1601323:verbs:fabric:verbs_devs_print():883 list of verbs devices found for FI_EP_MSG: libfabric:1601323:verbs:fabric:verbs_devs_print():887 #1 hfi1_0 - IPoIB addresses: libfabric:1601323:verbs:fabric:verbs_devs_print():897 10.10.8.58 libfabric:1601323:verbs:fabric:verbs_devs_print():897 fe80::211:7501:178:db8a libfabric:1601323:verbs:fabric:vrb_get_device_attrs():618 device hfi1_0: first found active port is 1 libfabric:1601323:verbs:fabric:vrb_get_device_attrs():563 XRC support unavailable in device: hfi1_0 libfabric:1601323:verbs:fabric:vrb_get_device_attrs():618 device hfi1_0: first found active port is 1 libfabric:1601323:core:core:ofi_register_provider():474 registering provider: verbs (113.20) libfabric:1601323:core:core:ofi_register_provider():474 registering provider: tcp (113.20) libfabric:1601323:core:core:ofi_register_provider():474 registering provider: sockets (113.20) libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ZE not supported libfabric:1601323:core:core:ofi_register_provider():474 registering provider: shm (113.20) libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_CUDA not supported libfabric:1601323:core:core:ofi_hmem_init():209 Hmem iface FI_HMEM_ROCR not supported libfabric:1601323:core:core:ze_hmem_dl_init():422 Failed to dlopen libze_loader.so libfabric:1601323:core:core:ofi_hmem_init():214 Failed to initialize hmem iface FI_HMEM_ZE: No data available libfabric:1601323:core:core:ofi_register_provider():474 registering provider: ofi_rxm (113.20) libfabric:1586090:core:core:ofi_register_provider():474 registering provider: psm2 (113.20) libfabric:1601323:core:core:ofi_register_provider():474 registering provider: psm2 (113.20) libfabric:1586090:psm3:core:fi_prov_ini():680 build options: VERSION=1101.0=11.1.0.0, HAVE_PSM3_SRC=1, PSM3_CUDA=0 libfabric:1586090:core:core:ofi_register_provider():474 registering provider: psm3 (1101.0) libfabric:1586090:core:core:ofi_register_provider():474 registering provider: mlx (1.4) libfabric:1586090:core:core:ofi_register_provider():474 registering provider: ofi_hook_noop (113.20) libfabric:1586090:core:core:fi_getinfo_():1138 Found provider with the highest priority psm2, must_use_util_prov = 0 libfabric:1586090:psm2:core:psmx2_getinfo():523 libfabric:1586090:psm2:core:psmx2_init_prov_info():254 RMA only instance included libfabric:1586090:psm2:core:psmx2_init_prov_info():268 TAG60 instance included libfabric:1586090:psm2:core:psmx2_init_prov_info():281 TAG64 instance included libfabric:1586090:psm2:core:psmx2_init_lib():257 PSM2 header version = (2, 1) libfabric:1586090:psm2:core:psmx2_init_lib():259 PSM2 library version = (2, 2) libfabric:1586090:psm2:core:psmx2_init_lib():264 PSM2 multi-ep feature not available or disabled. libfabric:1586090:psm2:core:psmx2_read_sysfs_int():286 /sys/class/infiniband/hfi1_0/ports/1/state: 4 libfabric:1586090:psm2:core:psmx2_read_sysfs_int():286 /sys/class/infiniband/hfi1_0/nctxts: 40 libfabric:1586090:psm2:core:psmx2_read_sysfs_int():286 /sys/class/infiniband/hfi1_0/nfreectxts: 32 libfabric:1586090:psm2:core:psmx2_update_hfi_info():427 hfi1 units: total 1, active 1; hfi1 contexts: total 40, free 32 libfabric:1586090:psm2:core:psmx2_update_hfi_info():439 Tx/Rx contexts: 1 in total, 1 available. libfabric:1586090:psm2:core:psmx2_alter_prov_info():449 3 instances available, 2 with CQ data flag set libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, mlx has been skipped. To use mlx, please, set FI_PROVIDER=mlx libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, psm3 has been skipped. To use psm3, please, set FI_PROVIDER=psm3 libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets libfabric:1601323:psm3:core:fi_prov_ini():680 build options: VERSION=1101.0=11.1.0.0, HAVE_PSM3_SRC=1, PSM3_CUDA=0 libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, shm has been skipped. To use shm, please, set FI_PROVIDER=shm libfabric:1586090:core:core:fi_getinfo_():1138 Found provider with the highest priority psm2, must_use_util_prov = 0 libfabric:1601323:core:core:ofi_register_provider():474 registering provider: psm3 (1101.0) libfabric:1586090:psm2:core:psmx2_getinfo():523 libfabric:1586090:psm2:core:psmx2_init_prov_info():268 TAG60 instance included libfabric:1586090:psm2:core:psmx2_alter_prov_info():449 1 instances available, 1 with CQ data flag set libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, mlx has been skipped. To use mlx, please, set FI_PROVIDER=mlx libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, psm3 has been skipped. To use psm3, please, set FI_PROVIDER=psm3 libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets libfabric:1586090:core:core:fi_getinfo_():1161 Since psm2 can be used, shm has been skipped. To use shm, please, set FI_PROVIDER=shm libfabric:1586090:psm2:core:psmx2_fabric():90 libfabric:1586090:core:core:fi_fabric_():1423 Opened fabric: psm2 libfabric:1586090:psm2:domain:psmx2_domain_open():356 libfabric:1586090:psm2:core:psmx2_init_tag_layout():171 use tag64: tag_mask: FFFFFFFFFFFFFFFF, data_mask: 0FFFFFFF libfabric:1586090:psm2:av:psmx2_av_open():1066 FI_AV_MAP asked, but force FI_AV_TABLE for shared AV libfabric:1586090:core:core:ofi_shm_map():153 shm_open failed libfabric:1586090:psm2:av:psmx2_av_open():1121 failed to map shared AV: FI_NAMED_AV_-1 libfabric:1586090:psm2:av:psmx2_av_open():1069 FI_AV_MAP asked, and granted libfabric:1586090:psm2:core:psmx2_trx_ctxt_alloc():298 uuid: 00FF00FF-0000-0000-0000-00FF00FF00FF libfabric:1586090:psm2:core:psmx2_trx_ctxt_alloc():303 ep_open_opts: unit=0 port=0 libfabric:1601323:core:core:ofi_register_provider():474 registering provider: mlx (1.4) libfabric:1601323:core:core:ofi_register_provider():474 registering provider: ofi_hook_noop (113.20) libfabric:1601323:core:core:fi_getinfo_():1138 Found provider with the highest priority psm2, must_use_util_prov = 0 libfabric:1601323:psm2:core:psmx2_getinfo():523 libfabric:1601323:psm2:core:psmx2_init_prov_info():254 RMA only instance included libfabric:1601323:psm2:core:psmx2_init_prov_info():268 TAG60 instance included libfabric:1601323:psm2:core:psmx2_init_prov_info():281 TAG64 instance included libfabric:1601323:psm2:core:psmx2_init_lib():257 PSM2 header version = (2, 1) libfabric:1601323:psm2:core:psmx2_init_lib():259 PSM2 library version = (2, 2) libfabric:1601323:psm2:core:psmx2_init_lib():264 PSM2 multi-ep feature not available or disabled. libfabric:1601323:psm2:core:psmx2_read_sysfs_int():286 /sys/class/infiniband/hfi1_0/ports/1/state: 4 libfabric:1601323:psm2:core:psmx2_read_sysfs_int():286 /sys/class/infiniband/hfi1_0/nctxts: 40 libfabric:1601323:psm2:core:psmx2_read_sysfs_int():286 /sys/class/infiniband/hfi1_0/nfreectxts: 8 libfabric:1601323:psm2:core:psmx2_update_hfi_info():427 hfi1 units: total 1, active 1; hfi1 contexts: total 40, free 8 libfabric:1601323:psm2:core:psmx2_update_hfi_info():439 Tx/Rx contexts: 1 in total, 1 available. libfabric:1601323:psm2:core:psmx2_alter_prov_info():449 3 instances available, 2 with CQ data flag set libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, mlx has been skipped. To use mlx, please, set FI_PROVIDER=mlx libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, psm3 has been skipped. To use psm3, please, set FI_PROVIDER=psm3 libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, shm has been skipped. To use shm, please, set FI_PROVIDER=shm libfabric:1601323:core:core:fi_getinfo_():1138 Found provider with the highest priority psm2, must_use_util_prov = 0 libfabric:1601323:psm2:core:psmx2_getinfo():523 libfabric:1601323:psm2:core:psmx2_init_prov_info():268 TAG60 instance included libfabric:1601323:psm2:core:psmx2_alter_prov_info():449 1 instances available, 1 with CQ data flag set libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, mlx has been skipped. To use mlx, please, set FI_PROVIDER=mlx libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, psm3 has been skipped. To use psm3, please, set FI_PROVIDER=psm3 libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets libfabric:1601323:core:core:fi_getinfo_():1161 Since psm2 can be used, shm has been skipped. To use shm, please, set FI_PROVIDER=shm libfabric:1601323:psm2:core:psmx2_fabric():90 libfabric:1601323:core:core:fi_fabric_():1423 Opened fabric: psm2 libfabric:1601323:psm2:domain:psmx2_domain_open():356 libfabric:1601323:psm2:core:psmx2_init_tag_layout():171 use tag64: tag_mask: FFFFFFFFFFFFFFFF, data_mask: 0FFFFFFF libfabric:1601323:psm2:av:psmx2_av_open():1066 FI_AV_MAP asked, but force FI_AV_TABLE for shared AV libfabric:1601323:core:core:ofi_shm_map():153 shm_open failed libfabric:1601323:psm2:av:psmx2_av_open():1121 failed to map shared AV: FI_NAMED_AV_-1 libfabric:1601323:psm2:av:psmx2_av_open():1069 FI_AV_MAP asked, and granted libfabric:1601323:psm2:core:psmx2_trx_ctxt_alloc():298 uuid: 00FF00FF-0000-0000-0000-00FF00FF00FF libfabric:1601323:psm2:core:psmx2_trx_ctxt_alloc():303 ep_open_opts: unit=0 port=0 libfabric:1586090:psm2:core:psmx2_trx_ctxt_alloc():333 epid: 00000000004a0d02 (tx+rx) libfabric:1586090:psm2:core:psmx2_am_init():116 epid 00000000004a0d02 libfabric:1586090:psm2:ep_data:psmx2_ep_optimize_ops():92 tagged ops optimized for op_flags=0 and directed receive libfabric:1586090:core:core:ofi_ns_add_local_name():370 Cannot add local name - name server uninitialized libfabric:1586090:psm2:ep_data:psmx2_ep_optimize_ops():92 tagged ops optimized for op_flags=0 and directed receive libfabric:1586090:psm2:ep_data:psmx2_ep_optimize_ops():87 tagged ops optimized for op_flags=0 and directed receive and av map libfabric:1601323:psm2:core:psmx2_trx_ctxt_alloc():333 epid: 00000000004b0502 (tx+rx) libfabric:1601323:psm2:core:psmx2_am_init():116 epid 00000000004b0502 libfabric:1601323:psm2:ep_data:psmx2_ep_optimize_ops():92 tagged ops optimized for op_flags=0 and directed receive libfabric:1601323:core:core:ofi_ns_add_local_name():370 Cannot add local name - name server uninitialized libfabric:1601323:psm2:ep_data:psmx2_ep_optimize_ops():92 tagged ops optimized for op_flags=0 and directed receive libfabric:1601323:psm2:ep_data:psmx2_ep_optimize_ops():87 tagged ops optimized for op_flags=0 and directed receive and av map Before wait