undefined reference to `cluster_sparse_solver' with 2021 mpiicc compiler

[segmentation_fault]

I am trying to compile this example:  cl_solver_sym_sp_0_based_c.c

Located in: /oneapi/2021.4/oneapi/mkl/latest/examples/examples_cluster_c.tgz

Here is the error message:

uxxxxx@login-2:~/solver$ mpiicc cl_solver_sym_sp_0_based_c.c
/home/uxxxxx/tmp/icckSqCzs.o: In function `main':
cl_solver_sym_sp_0_based_c.c:(.text+0x2cf): undefined reference to `cluster_sparse_solver'
cl_solver_sym_sp_0_based_c.c:(.text+0x3b2): undefined reference to `cluster_sparse_solver'
cl_solver_sym_sp_0_based_c.c:(.text+0x537): undefined reference to `cluster_sparse_solver'
cl_solver_sym_sp_0_based_c.c:(.text+0x5e6): undefined reference to `mkl_sparse_s_create_csr'
cl_solver_sym_sp_0_based_c.c:(.text+0x62f): undefined reference to `mkl_sparse_s_mv'
cl_solver_sym_sp_0_based_c.c:(.text+0x647): undefined reference to `mkl_sparse_destroy'
cl_solver_sym_sp_0_based_c.c:(.text+0x673): undefined reference to `mkl_sparse_destroy'
cl_solver_sym_sp_0_based_c.c:(.text+0x805): undefined reference to `cluster_sparse_solver'

[Gennady_F_Intel · Accepted solution]

please check what the MKL Linker Adviser suggests to compile/link for this case. You might specify the parameters in the drop-down lists and use these compiler and link lines in your building scripts. 

[segmentation_fault]

Thanks, got it to compile with this command per the link line advisor. Interesting it says -mkl=parralel is deprecated, though that is what the link advisor suggested.. Unfortunately, I am getting segmentation fault when trying to run it.

me@login-2:~/solver$ mpiicc -DMKL_ILP64 -mkl=parallel -L${MKLROOT}/lib/intel64 -lmkl_blacs_intelmpi_ilp64 -liomp5 -lpthread -lm -ldl cl_solver_sym_sp_0_based_c.c
icc: command line remark #10412: option '-mkl=parallel' is deprecated and will be removed in a future release. Please use the replacement option '-qmkl=parallel'
icc: warning #10315: specifying -lm before files may supersede the Intel(R) math library and affect performance

[Kirill_V_Intel]

Hi!

This likely means that you're still linking incorrectly. I'm afraid that recommendation with -mkl-parallel is not working as expected for this choice (which would mean that there is a bug in the recommendation by the link line advisor).

Just a quick check:

if we do ldd on the executable with your link line, I see

linux-vdso.so.1 => ...
libmkl_intel_lp64.so.1 => ...
libmkl_intel_thread.so.1 => ...
libmkl_core.so.1 => ...
libiomp5.so => ...
libmkl_blacs_intelmpi_ilp64.so.1 => ...
libpthread.so.0 => ...
libm.so.6 => ...
libdl.so.2 => ...
libmpifort.so.12 => ...
libmpi.so.12 => ...
...

So it links with lp64 interface.

Please tick the box "Link with Intel® oneMKL libraries explicitly" and use explicit link line.

That's what I get there for the link line (assuming you want dynamic linking):

-L${MKLROOT}/lib/intel64 -lmkl_intel_ilp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_ilp64 -liomp5 -lpthread -lm -ldl

Notice mkl_intel_ilp64!

Best,
Kirill

[segmentation_fault]

Thanks, that worked! I then was able to run the example on two cpus by using:

export OMP_NUM_THREADS=1

mpirun -np 2 ./a.out

[Gennady_F_Intel]

This issue has been resolved and we will no longer respond to this thread. If you require additional assistance from Intel, please start a new thread. Any further interaction in this thread will be considered community only. 

[segmentation_fault]

If anyone is interested, I modified the example to be able to read in the matrix (a) , ia, ja and b data from text files. Took me nearly two days to figure it out after battling many "segmentation faults" :-)..

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include "mpi.h"
#include "mkl.h"
#include "mkl_cluster_sparse_solver.h"

int main (void)
{
    /* Matrix data. */

//ndim = wc -l a.txt
//n = wc -l b.txt

// change in two places the zero based indexing to fortran: SPARSE_INDEX_BASE_ONE,  iparm[34] =  0

    MKL_INT n = 1219161;
    MKL_INT ndim = 46687911;

    MKL_INT ia[n+1];
    MKL_INT *ja = calloc ( ndim, sizeof ( MKL_INT ) );

    double *a = calloc ( ndim, sizeof ( double ) );

    MKL_INT mtype = -2;  /* set matrix type to "real symmetric indefinite matrix" */
    MKL_INT nrhs  =  1;  /* number of right hand sides. */

    double res, res0; /* RHS and solution vectors. */
    double *b = calloc ( n, sizeof ( double ) );
    double *x = calloc ( n, sizeof ( double ) );
    double *bs = calloc ( n, sizeof ( double ) );

//   printf ( "got here" );   return 1;

    /* Internal solver memory pointer pt
     *       32-bit:      int pt[64] or void *pt[64];
     *       64-bit: long int pt[64] or void *pt[64]; */
    void *pt[64] = { 0 };


    /* Cluster Sparse Solver control parameters. */
    MKL_INT iparm[64] = { 0 };
    MKL_INT maxfct, mnum, phase, msglvl, error;

    /* Auxiliary variables. */
    double   ddum; /* float dummy   */
    MKL_INT idum; /* Integer dummy. */
    MKL_INT i, j;
    int     mpi_stat = 0;
    int     argc = 0;
    int     comm, rank;
    char*   uplo;
    char**  argv;

/* -------------------------------------------------------------------- */
/* .. Init MPI.                                                         */
/* -------------------------------------------------------------------- */
    mpi_stat = MPI_Init( &argc, &argv );
    mpi_stat = MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    comm =  MPI_Comm_c2f( MPI_COMM_WORLD );
 
int row=0;
FILE *myfile;
int value;
double double_value;
  char line[80];

if ( rank == 0 ) {
  myfile = fopen ( "ia.txt", "r" );

while( !feof(myfile) && row < n + 1) {
    fgets( line, sizeof(line), myfile);
    sscanf( line, "%i",  &value );
   /* writing content to stdout */ 
//  printf ("%i\n",  value );
  ia[row]= value;
    row++;
}
fclose(myfile);

myfile = fopen ( "ja.txt", "r" );
row=0;
while( !feof(myfile) && row < ndim) {
    fgets( line, sizeof(line), myfile);
    sscanf( line, "%i",  &value );
   /* writing content to stdout */ 
//  printf ("%i\n",  value );
  ja[row]= value;
    row++;
//   if ( row %100000 ==0 ) { printf ( "Processed %i\n", row ); }

}

fclose(myfile);

myfile = fopen ( "a.txt", "r" );
row=0;
while( !feof(myfile) && row < ndim) {
    a[row] = 0.0;
    fgets( line, sizeof(line), myfile);
    sscanf( line, "%lf",  &double_value );
    a[row]= double_value;
    row++;
//   if ( row %100000 ==0 ) { printf ( "Processed %i\n", row ); }

}
fclose(myfile);
}

/* -------------------------------------------------------------------- */
/* .. Setup Cluster Sparse Solver control parameters.                                 */
/* -------------------------------------------------------------------- */
    iparm[ 0] =  1; /* Solver default parameters overriden with provided by iparm */
    iparm[ 1] =  2; /* Use METIS for fill-in reordering */
    iparm[ 5] =  0; /* Write solution into x */
    iparm[ 7] =  2; /* Max number of iterative refinement steps */
    iparm[ 9] = 13; /* Perturb the pivot elements with 1E-13 */
    iparm[10] =  0; /* Don't use nonsymmetric permutation and scaling MPS */
    iparm[12] =  1; /* Switch on Maximum Weighted Matching algorithm (default for non-symmetric) */
    iparm[17] = -1; /* Output: Number of nonzeros in the factor LU */
    iparm[18] = -1; /* Output: Mflops for LU factorization */
    iparm[27] =  0; /* Single precision mode of Cluster Sparse Solver */
    iparm[34] =  0; /* Cluster Sparse Solver use C-style indexing for ia and ja arrays */
    iparm[39] =  0; /* Input: matrix/rhs/solution stored on master */
    maxfct = 1; /* Maximum number of numerical factorizations. */
    mnum   = 1; /* Which factorization to use. */
    msglvl = 1; /* Print statistical information in file */
    error  = 0; /* Initialize error flag */

/* -------------------------------------------------------------------- */
/* .. Reordering and Symbolic Factorization. This step also allocates   */
/* all memory that is necessary for the factorization.                  */
/* -------------------------------------------------------------------- */
    phase = 11;
    cluster_sparse_solver ( pt, &maxfct, &mnum, &mtype, &phase,
                &n, a, ia, ja, &idum, &nrhs, iparm, &msglvl, &ddum, &ddum, &comm, &error );
    if ( error != 0 )
    {
        if ( rank == 0 ) printf ("\nERROR during symbolic factorization: %lli", (long long int)error);
        mpi_stat = MPI_Finalize();
        return 1;
    }
    if ( rank == 0 ) printf ("\nReordering completed ... ");

/* -------------------------------------------------------------------- */
/* .. Numerical factorization.                                          */
/* -------------------------------------------------------------------- */
    phase = 22;
    cluster_sparse_solver ( pt, &maxfct, &mnum, &mtype, &phase,
                &n, a, ia, ja, &idum, &nrhs, iparm, &msglvl, &ddum, &ddum, &comm, &error );
    if ( error != 0 )
    {
        if ( rank == 0 ) printf ("\nERROR during numerical factorization: %lli", (long long int)error);
        mpi_stat = MPI_Finalize();
        return 2;
    }
    if ( rank == 0 ) printf ("\nFactorization completed ... ");

/* -------------------------------------------------------------------- */
/* .. Back substitution and iterative refinement.                       */
/* -------------------------------------------------------------------- */
    phase = 33;

row = 0;
myfile = fopen ( "b.txt", "r" );

while( !feof(myfile) && row < n ) {
   fgets( line, sizeof(line), myfile);
   sscanf( line, "%lf",  &double_value );
  b[row]= double_value;
   x[row] = 0.0;
    row++;
}

fclose(myfile);

     if ( rank == 0 ) printf ("\nSolving system...");
    cluster_sparse_solver ( pt, &maxfct, &mnum, &mtype, &phase,
                &n, a, ia, ja, &idum, &nrhs, iparm, &msglvl, b, x, &comm, &error );
    if ( error != 0 )
    {
        if ( rank == 0 ) printf ("\nERROR during solution: %lli", (long long int)error);
        mpi_stat = MPI_Finalize();
        return 4;
    }

    if ( rank == 0 )
    {
        printf ("\nThe solution of the system is: ");
        for ( j = 0; j < n; j++ )
        {
            printf ( "\n x [%lli] = % f", (long long int)j, x[j] );
        }
        /*  Compute residual */
        struct matrix_descr descrA;
        sparse_matrix_t csrA;
        sparse_status_t status;
        status = mkl_sparse_d_create_csr( &csrA, SPARSE_INDEX_BASE_ONE, n, n, ia, ia+1, ja, a);
        if (status != SPARSE_STATUS_SUCCESS) {
            printf("Error in mkl_sparse_s_create_csr\n");
            mkl_sparse_destroy( csrA );
            goto final;
        }
        descrA.type = SPARSE_MATRIX_TYPE_SYMMETRIC;
        descrA.mode = SPARSE_FILL_MODE_UPPER;
        descrA.diag = SPARSE_DIAG_NON_UNIT;
        status = mkl_sparse_d_mv( SPARSE_OPERATION_NON_TRANSPOSE, 1.0, csrA, descrA, x, 0.0, bs);
        if (status != SPARSE_STATUS_SUCCESS) {
            printf("Error in mkl_sparse_s_mv\n");
            mkl_sparse_destroy( csrA );
            goto final;
        }
        mkl_sparse_destroy( csrA );

        res  = 0.0;
        res0 = 0.0;
        for ( j = 0; j < n; j++ )
        {
//            printf ( "\nCalculating %f - %f\n" , bs[j], b[j] );
            res  += (bs[j] - b[j]) * (bs[j] - b[j]);
            res0 += b[j] * b[j];
       }
        res = sqrt ( res ) / sqrt ( res0 );
        printf ( "\nRelative residual = %e\n", res );
    }

/* -------------------------------------------------------------------- */
/* .. Termination and release of memory. */
/* -------------------------------------------------------------------- */
    phase = -1; /* Release internal memory. */
    cluster_sparse_solver ( pt, &maxfct, &mnum, &mtype, &phase,
                &n, &ddum, ia, ja, &idum, &nrhs, iparm, &msglvl, &ddum, &ddum, &comm, &error );
    if ( error != 0 )
    {
        if ( rank == 0 ) printf ("\nERROR during release memory: %lli", (long long int)error);
        goto final;
    }
    /* Check residual */
    if(rank == 0)
        {
            if ( res > 1e-5 )
        {
            printf ("\nError: residual is too high!\n");
            error = 5;
            goto final;
        }
        }
final:
    if ( rank == 0 )
    {
        if ( error != 0 )
        {
            printf("\n TEST FAILED\n");
        } else {
            printf("\n TEST PASSED\n");
        }
    }
    mpi_stat = MPI_Finalize();
    return error;
}