alex@thor|~/mpi_mic> setenv I_MPI_MIC 1 alex@thor|~/mpi_mic> setenv I_MPI_MIC_PREFIX $I_MPI_ROOT/mic/bin/ alex@thor|~/mpi_mic> setenv I_MPI_DEBUG 100 alex@thor|~/mpi_mic> mpirun -ppn 1 -n 2 -hosts thor,mic0 IMB-MPI1 pingpong [0] MPI startup(): Intel(R) MPI Library, Version 5.0 Update 3 Build 20150128 (build id: 11250) [0] MPI startup(): Copyright (C) 2003-2015 Intel Corporation. All rights reserved. [0] MPI startup(): Multi-threaded optimized library [0] MPID_nem_impi_create_numa_nodes_map(): NUMA map->self_id = 0 [0] MPID_nem_impi_create_numa_nodes_map(): NUMA map->devices_num = 2 [0] MPID_nem_impi_create_numa_nodes_map(): NUMA map->nodes_num = 2 [0] MPID_nem_impi_create_numa_nodes_map(): NUMA map->devices: [0] MPID_nem_impi_create_numa_nodes_map(): mic0:1 [0] MPID_nem_impi_create_numa_nodes_map(): mic1:1 [0] MPID_nem_impi_create_numa_nodes_map(): NUMA map->distances: [0] MPID_nem_impi_create_numa_nodes_map(): 0 -> 0 = 10 [0] MPID_nem_impi_create_numa_nodes_map(): 0 -> 1 = 21 [1] MPID_nem_impi_create_numa_nodes_map(): Fetching extra numa information from /etc/ofed-mic.map [0] MPI startup(): RLIMIT_MEMLOCK too small [0] MPI startup(): fabric dapl failed: will try use ofa fabric [0] MPI startup(): RLIMIT_MEMLOCK too small [1] my_dlopen(): trying to dlopen: libdat2.so.2 [1] MPID_nem_dapl_classify_all_providers_20(): start provider classification [1] I_MPI_dlopen_dat(): trying to load default dat library: libdat2.so.2 [1] my_dlopen(): trying to dlopen: libdat2.so.2 [1] MPID_nem_dapl_classify_all_providers_20(): finish provider classification [1] dapl_auto_provider_selection(): successful provider classification [1] MPI startup(): DAPL ofa-v2-mic0 direct copy threshold=4272 [1] MPI startup(): DAPL ofa-v2-scif0 direct copy threshold=262144 [1] MPI startup(): DAPL ofa-v2-mic0 rdma_rndv_buf_align_mask=63 [1] MPI startup(): DAPL ofa-v2-scif0 rdma_rndv_buf_align_mask=63 [1] MPI startup(): DAPL ofa-v2-mic0 rdma_rndv_copy_with_align_threshold=65536 [1] MPI startup(): DAPL ofa-v2-scif0 rdma_rndv_copy_with_align_threshold=65536 [1] MPI startup(): DAPL ofa-v2-mic0 rdma_max_msg_size=16777216 [1] MPI startup(): DAPL ofa-v2-scif0 rdma_max_msg_size=16777216 [1] MPI startup(): DAPL ofa-v2-mic0 rdma_max_fragment_num=16 [1] MPI startup(): DAPL ofa-v2-scif0 rdma_max_fragment_num=16 [1] MPI startup(): DAPL ofa-v2-mic0 req_evd_qlen=2000 [1] MPI startup(): DAPL ofa-v2-scif0 req_evd_qlen=2000 [1] MPI startup(): DAPL ofa-v2-mic0 conn_evd_qlen=36 [1] MPI startup(): DAPL ofa-v2-scif0 conn_evd_qlen=36 [1] MPI startup(): DAPL ofa-v2-mic0 dapl_connection_timeout=4294967295 [1] MPI startup(): DAPL ofa-v2-scif0 dapl_connection_timeout=4294967295 [1] MPI startup(): DAPL ofa-v2-mic0 dapl_disconnection_timeout=10000000 [1] MPI startup(): DAPL ofa-v2-scif0 dapl_disconnection_timeout=10000000 [1] MPI startup(): DAPL ofa-v2-mic0 service_id=0 [1] MPI startup(): DAPL ofa-v2-scif0 service_id=0 [1] MPI startup(): DAPL ofa-v2-mic0 create_conn_qual=-1 [1] MPI startup(): DAPL ofa-v2-scif0 create_conn_qual=-1 [1] MPI startup(): DAPL ofa-v2-mic0 rdma_use_mreg_cache=1 [1] MPI startup(): DAPL ofa-v2-scif0 rdma_use_mreg_cache=1 [1] MPI startup(): DAPL ofa-v2-mic0 rtc_use_avl_tree=0 [1] MPI startup(): DAPL ofa-v2-scif0 rtc_use_avl_tree=0 [1] MPI startup(): DAPL ofa-v2-mic0 rtc_max_cache_entries=1024 [1] MPI startup(): DAPL ofa-v2-scif0 rtc_max_cache_entries=1024 [1] MPI startup(): DAPL ofa-v2-mic0 rtc_max_memory_size=0 [1] MPI startup(): DAPL ofa-v2-scif0 rtc_max_memory_size=0 [1] MPI startup(): DAPL ofa-v2-mic0 conn_desired_num=256 [1] MPI startup(): DAPL ofa-v2-scif0 conn_desired_num=256 [1] MPI startup(): DAPL ofa-v2-mic0 disconn_desired_num=1024 [1] MPI startup(): DAPL ofa-v2-scif0 disconn_desired_num=1024 [1] DAPL startup(): trying to open first DAPL provider from I_MPI_DAPL_PROVIDER_LIST: ofa-v2-mic0 [1] I_MPI_dlopen_dat(): trying to load default dat library: libdat2.so.2 [1] my_dlopen(): trying to dlopen: libdat2.so.2 [1] MPI startup(): DAPL provider ofa-v2-mic0 [1] MPI startup(): DAPL provider ofa-v2-mic0 will use RDMA_READ direct copy protocol [1] DAPL startup(): trying to open secondary (2) DAPL provider from I_MPI_DAPL_PROVIDER_LIST: ofa-v2-scif0 [1] MPI startup(): secondary DAPL provider ofa-v2-scif0 [1] MPI startup(): provider ofa-v2-scif0 is topology bound [1] MPI startup(): secondary DAPL provider ofa-v2-scif0 will use RDMA_READ direct copy protocol [1] MPI startup(): dapl data transfer mode [0] MPI startup(): fabric ofa failed: will try use tcp fabric [0] MPI startup(): tcp data transfer mode