# FI_LOG_LEVEL: String
# Specify logging level: warn, trace, info, debug (default: warn)

# FI_LOG_PROV: String
# Specify specific provider to log (default: all)

# FI_LOG_SUBSYS: String
# Specify specific subsystem to log (default: all)

# FI_PERF_CNTR: String
# Performance counter to analyze (default: cpu_instr). Options: cpu_instr, cpu_cycles.

# FI_HOOK: String
# Intercept calls to underlying provider and apply the specified functionality to them.  Hook option: perf (gather performance data)

# FI_MR_CACHE_MAX_SIZE: size_t
# Defines the total number of bytes for all memory regions that may be tracked by the MR cache. Setting this will reduce the amount of memory not actively in use that may be registered. (default: total memory / number of cpu cores / 2)

# FI_MR_CACHE_MAX_COUNT: size_t
# Defines the total number of memory regions that may be store in the cache.  Setting this will reduce the number of registered regions, regardless of their size, stored in the cache.  Setting this to zero will disable MR caching.  (default: 1024)

# FI_MR_CACHE_MONITOR: String
# Define a default memory registration monitor. The monitor checks for virtual to physical memory address changes.  Options are: userfaultfd, memhooks and disabled.  Userfaultfd is a Linux kernel feature. Memhooks operates by intercepting memory allocation and free calls.  Userfaultfd is the default if available on the system. 'disabled' option disables memory caching.

# FI_MR_CUDA_CACHE_MONITOR_ENABLED: Boolean (0/1, on/off, true/false, yes/no)
# Enable or disable the CUDA cache memory monitor.Monitor is enabled by default.

# FI_MR_ROCR_CACHE_MONITOR_ENABLED: Boolean (0/1, on/off, true/false, yes/no)
# Enable or disable the ROCR cache memory monitor. Monitor is enabled by default.

# FI_PROVIDER: String
# Only use specified provider (default: all available)

# FI_FORK_UNSAFE: Boolean (0/1, on/off, true/false, yes/no)
# Whether use of fork() may be unsafe for some providers (default: no). Setting this to yes could improve performance at the expense of making fork() potentially unsafe

# FI_UNIVERSE_SIZE: size_t
# Defines the maximum number of processes that will be used by distribute OFI application. The provider uses this to optimize resource allocations (default: provider specific)

# FI_PROVIDER_PATH: String
# Search for providers in specific path (default: /usr/lib64/libfabric)

# FI_PSM3_NAME_SERVER: Boolean (0/1, on/off, true/false, yes/no)
# psm3: Whether to turn on the name server or not (default: yes)

# FI_PSM3_TAGGED_RMA: Boolean (0/1, on/off, true/false, yes/no)
# psm3: Whether to use tagged messages for large size RMA or not (default: yes)

# FI_PSM3_UUID: String
# psm3: Unique Job ID required by the fabric

# FI_PSM3_DELAY: Integer
# psm3: Delay (seconds) before finalization (for debugging)

# FI_PSM3_TIMEOUT: Integer
# psm3: Timeout (seconds) for gracefully closing the PSM3 endpoint

# FI_PSM3_CONN_TIMEOUT: Integer
# psm3: Timeout (seconds) for establishing connection between two PSM3 endpoints

# FI_PSM3_PROG_INTERVAL: Integer
# psm3: Interval (microseconds) between progress calls made in the progress thread (default: 1 if affinity is set, 1000 if not)

# FI_PSM3_PROG_AFFINITY: String
# psm3: When set, specify the set of CPU cores to set the progress thread affinity to. The format is <start>[:<end>[:<stride>]][,<start>[:<end>[:<stride>]]]*, where each triplet <start>:<end>:<stride> defines a block of core_ids. Both <start> and <end> can be either the core_id (when >=0) or core_id - num_cores (when <0). (default: affinity not set)

# FI_PSM3_INJECT_SIZE: Integer
# psm3: Maximum message size for fi_inject and fi_tinject (default: 64).

# FI_PSM3_LOCK_LEVEL: Integer
# psm3: How internal locking is used. 0 means no locking. (default: 2).

# FI_PSM3_LAZY_CONN: Boolean (0/1, on/off, true/false, yes/no)
# psm3: Whether to force lazy connection mode. (default: no).

# FI_PSM3_DISCONNECT: Boolean (0/1, on/off, true/false, yes/no)
# psm3: Whether to issue disconnect request when process ends (default: no).

# FI_PSM3_TAG_LAYOUT: String
# psm3: How the 96 bit PSM3 tag is organized: tag60 means 32/4/60 for data/flags/tag;tag64 means 4/28/64 for flags/data/tag (default: tag60).

# FI_PSM2_NAME_SERVER: Boolean (0/1, on/off, true/false, yes/no)
# psm2: Whether to turn on the name server or not (default: yes)

# FI_PSM2_TAGGED_RMA: Boolean (0/1, on/off, true/false, yes/no)
# psm2: Whether to use tagged messages for large size RMA or not (default: yes)

# FI_PSM2_UUID: String
# psm2: Unique Job ID required by the fabric

# FI_PSM2_DELAY: Integer
# psm2: Delay (seconds) before finalization (for debugging)

# FI_PSM2_TIMEOUT: Integer
# psm2: Timeout (seconds) for gracefully closing the PSM2 endpoint

# FI_PSM2_CONN_TIMEOUT: Integer
# psm2: Timeout (seconds) for establishing connection between two PSM2 endpoints

# FI_PSM2_PROG_INTERVAL: Integer
# psm2: Interval (microseconds) between progress calls made in the progress thread (default: 1 if affinity is set, 1000 if not)

# FI_PSM2_PROG_AFFINITY: String
# psm2: When set, specify the set of CPU cores to set the progress thread affinity to. The format is <start>[:<end>[:<stride>]][,<start>[:<end>[:<stride>]]]*, where each triplet <start>:<end>:<stride> defines a block of core_ids. Both <start> and <end> can be either the core_id (when >=0) or core_id - num_cores (when <0). (default: affinity not set)

# FI_PSM2_INJECT_SIZE: Integer
# psm2: Maximum message size for fi_inject and fi_tinject (default: 64).

# FI_PSM2_LOCK_LEVEL: Integer
# psm2: How internal locking is used. 0 means no locking. (default: 2).

# FI_PSM2_LAZY_CONN: Boolean (0/1, on/off, true/false, yes/no)
# psm2: Whether to force lazy connection mode. (default: no).

# FI_PSM2_DISCONNECT: Boolean (0/1, on/off, true/false, yes/no)
# psm2: Whether to issue disconnect request when process ends (default: no).

# FI_PSM2_TAG_LAYOUT: String
# psm2: How the 96 bit PSM2 tag is organized: tag60 means 32/4/60 for data/flags/tag;tag64 means 4/28/64 for flags/data/tag (default: tag60).

# FI_SHM_SAR_THRESHOLD: size_t
# shm: Max size to use for alternate SAR protocol if CMA 			 is not available before switching to mmap protocol 			 Default: SIZE_MAX (18446744073709551615)

# FI_SHM_TX_SIZE: size_t
# shm: Max number of outstanding tx operations 			 Default: 1024

# FI_SHM_RX_SIZE: size_t
# shm: Max number of outstanding rx operations 			 Default: 1024

# FI_OFI_RXM_BUFFER_SIZE: size_t
# ofi_rxm: Defines the transmit buffer size / inject size (default: 16 KB). Eager protocol would be used to transmit messages of size less than eager limit (FI_OFI_RXM_BUFFER_SIZE - RxM header size (64 B)). Any message whose size is greater than eager limit would be transmitted via rendezvous or SAR (Segmentation And Reassembly) protocol depending on the value of FI_OFI_RXM_SAR_LIMIT). Also, transmit data  would be copied up to eager limit.

# FI_OFI_RXM_COMP_PER_PROGRESS: Integer
# ofi_rxm: Defines the maximum number of MSG provider CQ entries (default: 1) that would be read per progress (RxM CQ read).

# FI_OFI_RXM_SAR_LIMIT: size_t
# ofi_rxm: Set this environment variable to enable and control RxM SAR (Segmentation And Reassembly) protocol (default: 128 KB). This value should be set greater than  eager limit (FI_OFI_RXM_BUFFER_SIZE - RxM protocol header size (64 B)) for SAR to take effect. Messages of size greater than this would be transmitted via rendezvous protocol.

# FI_OFI_RXM_USE_SRX: Boolean (0/1, on/off, true/false, yes/no)
# ofi_rxm: Set this environment variable to control the RxM receive path. If this variable set to 1 (default: 0), the RxM uses Shared Receive Context. This mode improves memory consumption, but it may increase small message latency as a side-effect.

# FI_OFI_RXM_TX_SIZE: size_t
# ofi_rxm: Defines default tx context size (default: 65536).

# FI_OFI_RXM_RX_SIZE: size_t
# ofi_rxm: Defines default rx context size (default: 65536).

# FI_OFI_RXM_MSG_TX_SIZE: size_t
# ofi_rxm: Defines FI_EP_MSG tx size that would be requested (default: 128). Setting this to 0 would get default value defined by the MSG provider.

# FI_OFI_RXM_MSG_RX_SIZE: size_t
# ofi_rxm: Defines FI_EP_MSG rx size that would be requested (default: 128). Setting this to 0 would get default value defined by the MSG provider.

# FI_OFI_RXM_CM_PROGRESS_INTERVAL: Integer
# ofi_rxm: Defines the number of microseconds to wait between function calls to the connection management progression functions during fi_cq_read calls. Higher values may decrease noise during cq polling, but may result in longer connection establishment times. (default: 10000).

# FI_OFI_RXM_CQ_EQ_FAIRNESS: Integer
# ofi_rxm: Defines the maximum number of message provider CQ entries that can be consecutively read across progress calls without checking to see if the CM progress interval has been reached. (default: 128).

# FI_OFI_RXM_DATA_AUTO_PROGRESS: Boolean (0/1, on/off, true/false, yes/no)
# ofi_rxm: Force auto-progress for data transfers even if app requested manual progress (default: false/no).

# FI_OFI_RXM_USE_RNDV_WRITE: Boolean (0/1, on/off, true/false, yes/no)
# ofi_rxm: Set this environment variable to control the  RxM Rendezvous protocol.  If set (1), RxM will use RMA writes rather than RMA reads during Rendezvous transactions. (default: false/no).

# FI_OFI_RXM_ENABLE_DYN_RBUF: Boolean (0/1, on/off, true/false, yes/no)
# ofi_rxm: Enable support for dynamic receive buffering, if available by the message endpoint provider. This allows direct placement of received messages into application buffers, bypassing RxM bounce buffers.  This feature targets using tcp sockets for the message transport.  (default: false)

# FI_OFI_RXM_ENABLE_DIRECT_SEND: Boolean (0/1, on/off, true/false, yes/no)
# ofi_rxm: Enable support to pass application buffers directly to the core provider when possible.  This avoids copying application buffers through bounce buffers before passing them to the core provider.  This feature targets small to medium size message transfers over the tcp provider.  (default: false)

# FI_OFI_RXM_DEF_WAIT_OBJ: String
# ofi_rxm: Specifies the default wait object used for blocking operations (e.g. fi_cq_sread).  Supported values are: fd and pollfd (default: fd).

# FI_OFI_RXM_DEF_TCP_WAIT_OBJ: String
# ofi_rxm: See def_wait_obj for description.  If set, this overrides the def_wait_obj when running over the tcp provider.

# FI_VERBS_TX_SIZE: Integer
# verbs: Default maximum tx context size (default: 384)

# FI_VERBS_RX_SIZE: Integer
# verbs: Default maximum rx context size (default: 384)

# FI_VERBS_TX_IOV_LIMIT: Integer
# verbs: Default maximum tx iov_limit (default: 4)

# FI_VERBS_RX_IOV_LIMIT: Integer
# verbs: Default maximum rx iov_limit (default: 4)

# FI_VERBS_INLINE_SIZE: Integer
# verbs: Default maximum inline size. Actual inject size returned in fi_info may be greater (default: 256)

# FI_VERBS_MIN_RNR_TIMER: Integer
# verbs: Set min_rnr_timer QP attribute (0 - 31) (default: 12)

# FI_VERBS_USE_ODP: Boolean (0/1, on/off, true/false, yes/no)
# verbs: Enable on-demand paging memory registrations, if supported.  This is currently required to register DAX file system mmapped memory. (default: 0)

# FI_VERBS_PREFER_XRC: Boolean (0/1, on/off, true/false, yes/no)
# verbs: Order XRC transport fi_infos ahead of RC.  Default orders RC first.  This setting must usually be combined with setting FI_OFI_RXM_USE_SRX.  See fi_verbs.7 man page. (default: 0)

# FI_VERBS_XRCD_FILENAME: String
# verbs: A file to associate with the XRC domain. (default: /tmp/verbs_xrcd)

# FI_VERBS_CQREAD_BUNCH_SIZE: Integer
# verbs: The number of entries to be read from the verbs completion queue at a time (default: 8)

# FI_VERBS_GID_IDX: Integer
# verbs: Set which gid index to use attribute (0 - 255) (default: 0)

# FI_VERBS_DEVICE_NAME: String
# verbs: The prefix or the full name of the verbs device to use (default: )

# FI_VERBS_IFACE: String
# verbs: The prefix or the full name of the network interface associated with the verbs device (default: )

# FI_VERBS_DGRAM_USE_NAME_SERVER: Boolean (0/1, on/off, true/false, yes/no)
# verbs: The option that enables/disables OFI Name Server thread used to resolve IP-addresses to provider specific addresses. If MPI is used, the NS is disabled by default. (default: 1)

# FI_VERBS_DGRAM_NAME_SERVER_PORT: Integer
# verbs: The port on which the name server thread listens incoming requests. (default: 5678)

# FI_OFI_MRAIL_CONFIG: String
# ofi_mrail: Comma separated list of '<max_size>:<policy>' pairs, with <max_size> in ascending order and <policy> being fixed, round-robin, or striping

# FI_OFI_MRAIL_ADDR_STRC: String
# ofi_mrail: Deprecated. Replaced by FI_OFI_MRAIL_ADDR.

# FI_OFI_MRAIL_ADDR: String
# ofi_mrail: Comma separated list of rail addresses (FI_ADDR_STR, host name, IP address, or netdev interface name)

# FI_OFI_RXD_SPIN_COUNT: Integer
# ofi_rxd: Number of iterations to receive packets (0 - infinite)

# FI_OFI_RXD_RETRY: Boolean (0/1, on/off, true/false, yes/no)
# ofi_rxd: Toggle packet retrying (default: yes)

# FI_OFI_RXD_MAX_PEERS: Integer
# ofi_rxd: Maximum number of peers to track (default: 1024)

# FI_OFI_RXD_MAX_UNACKED: Integer
# ofi_rxd: Maximum number of packets to send at once (default: 128)

# FI_EFA_RX_WINDOW_SIZE: Integer
# efa: Defines the maximum window size that a receiver will return for matched large messages. (Default: 128).

# FI_EFA_TX_MAX_CREDITS: Integer
# efa: Defines the maximum number of credits a sender requests from a receiver (Default: 64).

# FI_EFA_TX_MIN_CREDITS: Integer
# efa: Defines the minimum number of credits a sender requests from a receiver (Default: 32).

# FI_EFA_TX_QUEUE_SIZE: Integer
# efa: Defines the maximum number of unacknowledged sends with the NIC.

# FI_EFA_ENABLE_SHM_TRANSFER: Integer
# efa: Enable using SHM provider to provide the communication between processes on the same system. (Default: 1)

# FI_EFA_USE_DEVICE_RDMA: Integer
# efa: whether to use device's RDMA functionality for one-sided and two-sided transfer.

# FI_EFA_USE_ZCPY_RX: Integer
# efa: Enables the use of application's receive buffers in place of bounce-buffers when feasible. (Default: 1)

# FI_EFA_ZCPY_RX_SEED: Integer
# efa: Defines the number of bounce-buffers the provider will prepost during EP initialization.  (Default: 0)

# FI_EFA_SHM_AV_SIZE: Integer
# efa: Defines the maximum number of entries in SHM provider's address vector (Default 128).

# FI_EFA_SHM_MAX_MEDIUM_SIZE: Integer
# efa: Defines the switch point between small/medium message and large message. The message larger than this switch point will be transferred with large message protocol (Default 4096).

# FI_EFA_RECVWIN_SIZE: Integer
# efa: Defines the size of sliding receive window. (Default: 16384)

# FI_EFA_READCOPY_POOL_SIZE: Integer
# efa: Defines the size of readcopy packet pool size. (Default: 256)

# FI_EFA_CQ_SIZE: Integer
# efa: Define the size of completion queue. (Default: 8192)

# FI_EFA_MR_CACHE_ENABLE: Boolean (0/1, on/off, true/false, yes/no)
# efa: Enables using the mr cache and in-line registration instead of a bounce buffer for iov's larger than max_memcpy_size. Defaults to true. When disabled, only uses a bounce buffer.

# FI_EFA_MR_MAX_CACHED_COUNT: size_t
# efa: Sets the maximum number of memory registrations that can be cached at any time.

# FI_EFA_MR_MAX_CACHED_SIZE: size_t
# efa: Sets the maximum amount of memory that cached memory registrations can hold onto at any time.

# FI_EFA_MAX_MEMCPY_SIZE: size_t
# efa: Threshold size switch between using memory copy into a pre-registered bounce buffer and memory registration on the user buffer. (Default: 4096)

# FI_EFA_MTU_SIZE: size_t
# efa: Override the MTU size of the device.

# FI_EFA_TX_SIZE: size_t
# efa: Set the maximum number of transmit operations before the provider returns -FI_EAGAIN. For only the RDM endpoint, this parameter will cause transmit operations to be queued when this value is set higher than the default and the transmit queue is full.

# FI_EFA_RX_SIZE: size_t
# efa: Set the maximum number of receive operations before the provider returns -FI_EAGAIN.

# FI_EFA_TX_IOV_LIMIT: size_t
# efa: Maximum transmit iov_limit.

# FI_EFA_RX_IOV_LIMIT: size_t
# efa: Maximum receive iov_limit.

# FI_EFA_RX_COPY_UNEXP: Boolean (0/1, on/off, true/false, yes/no)
# efa: Enables the use of a separate pool of bounce-buffers to copy unexpected messages out of the pre-posted receive buffers. (Default: 1)

# FI_EFA_RX_COPY_OOO: Boolean (0/1, on/off, true/false, yes/no)
# efa: Enables the use of a separate pool of bounce-buffers to copy out-of-order RTM packets out of the pre-posted receive buffers. (Default: 1)

# FI_EFA_MAX_TIMEOUT: Integer
# efa: Set the maximum timeout (us) for backoff to a peer after a receiver not ready error. (Default: 1000000)

# FI_EFA_TIMEOUT_INTERVAL: Integer
# efa: Set the time interval (us) for the base timeout to use for exponential backoff to a peer after a receiver not ready error. (Default: 0 [random])

# FI_EFA_EFA_CQ_READ_SIZE: size_t
# efa: Set the number of EFA completion entries to read for one loop for one iteration of the progress engine. (Default: 50)

# FI_EFA_SHM_CQ_READ_SIZE: size_t
# efa: Set the number of SHM completion entries to read for one loop for one iteration of the progress engine. (Default: 50)

# FI_EFA_INTER_MAX_MEDIUM_MESSAGE_SIZE: Integer
# efa: The maximum message size for inter EFA medium message protocol (Default 65536).

# FI_EFA_INTER_MIN_READ_MESSAGE_SIZE: Integer
# efa: The minimum message size for inter EFA read message protocol. If instance support RDMA read, messages whose size is larger than this value will be sent by read message protocol (Default 1048576).

# FI_EFA_INTER_MIN_READ_WRITE_SIZE: Integer
# efa: The mimimum message size for inter EFA write to use read write protocol. If firmware support RDMA read, and FI_EFA_USE_DEVICE_RDMA is 1, write requests whose size is larger than this value will use the read write protocol (Default 65536).

# FI_EFA_INTER_READ_SEGMENT_SIZE: Integer
# efa: Calls to RDMA read is segmented using this value.

# FI_UDP_IFACE: String
# udp: Specify interface name

# FI_SOCKETS_PE_WAITTIME: Integer
# sockets: How many milliseconds to spin while waiting for progress

# FI_SOCKETS_CONN_TIMEOUT: Integer
# sockets: How many milliseconds to wait for one connection establishment

# FI_SOCKETS_MAX_CONN_RETRY: Integer
# sockets: Number of connection retries before reporting as failure

# FI_SOCKETS_DEF_CONN_MAP_SZ: Integer
# sockets: Default connection map size

# FI_SOCKETS_DEF_AV_SZ: Integer
# sockets: Default address vector size

# FI_SOCKETS_DEF_CQ_SZ: Integer
# sockets: Default completion queue size

# FI_SOCKETS_DEF_EQ_SZ: Integer
# sockets: Default event queue size

# FI_SOCKETS_PE_AFFINITY: String
# sockets: If specified, bind the progress thread to the indicated range(s) of Linux virtual processor ID(s). This option is currently not supported on OS X and Windows. Usage: id_start[-id_end[:stride]][,]

# FI_SOCKETS_KEEPALIVE_ENABLE: Boolean (0/1, on/off, true/false, yes/no)
# sockets: Enable keepalive support

# FI_SOCKETS_KEEPALIVE_TIME: Integer
# sockets: Idle time in seconds before sending the first keepalive probe

# FI_SOCKETS_KEEPALIVE_INTVL: Integer
# sockets: Time in seconds between individual keepalive probes

# FI_SOCKETS_KEEPALIVE_PROBES: Integer
# sockets: Maximum number of keepalive probes sent before dropping the connection

# FI_SOCKETS_IFACE: String
# sockets: Specify interface name

# FI_TCP_IFACE: String
# tcp: Specify interface name

# FI_TCP_PORT_LOW_RANGE: Integer
# tcp: define port low range

# FI_TCP_PORT_HIGH_RANGE: Integer
# tcp: define port high range

# FI_TCP_NODELAY: Boolean (0/1, on/off, true/false, yes/no)
# tcp: overrides default TCP_NODELAY socket setting