MAGMAmic fails with ENXIO in dgesv_mic routine on Xeon Phi

Dimitar_S_1 · ‎09-27-2017

Hello,

I am trying to use the magma_dgesv_mic routine on Xeon Phi. I have compiled magma successfully and I am able to run the testing_dgesv_mic executable. However when I implement it in my code it doesn't work and gives me an error.

I am able to compile the code and run it but every time I receive an error:



	scif_writeto failed for mic-1 with err 6

	writeto: ENXIO

	scif_readfrom failed for mic-1 with err 6

	scif_writeto failed for mic-1 with err 6

	scif_readfrom failed for mic-1 with err 6

	scif_writeto failed for mic-1 with err 6

There is a topic on the same issue here: https://software.intel.com/en-us/forums/intel-many-integrated-core/topic/516283

However the topic starter only reports that he had solved it, but not how.

Does anyone knows what might be the issue?

This is my code.



	 /********************************************************                                                               

	  * Function Direct                                                                                                      

	  * Calculates the ga in every airfoil and every interval                                                                

	  * Creator Dimitar Slavchev                                                                                             

	  * current date 07.06.2011 BC                                                                                           

	  *******************************************************/

// MAGMA headers
#include "magma.h"
#include "magma_lapack.h"
#include "magma_types.h"

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <time.h>
#include <omp.h>

// My Headers
#include "constants.h"
#include "global.h"
#include "mymath.h"
#include "vort.h"

#define TESTING_MALLOC_HOST( ptr, type, size )                                \
     if ( MAGMA_SUCCESS !=                                                     \
             magma_malloc_pinned( (void**) &ptr, (size)*sizeof(type) ) ) {     \
         fprintf( stderr, "!!!! magma_malloc_pinned failed for: %s\n", #ptr ); \
         exit(-1);                                                             \
     }

#define TESTING_MALLOC_DEV( ptr, type, size )                              \
     if ( MAGMA_SUCCESS !=                                                  \
             magma_malloc( (void**) &ptr, (size)*sizeof(type) ) ) {         \
         fprintf( stderr, "!!!! magma_malloc failed for: %s\n", #ptr );     \
         magma_finalize();                                                  \
         exit(-1);                                                          \
     }

void direct(int* N, double **PA, Coord **points)
{
     magma_int_t     i ,j ,k ,l, ir, irr,
                     md = suma(N , NUM_AIRFOILS) - NUM_AIRFOILS,
                     m;

     double *h_D, *h_sv, // Host D and sv
             *A,
             *d_D, *d_sv; // D and sv on the device

     magma_int_t     *IPIV, info;
     magma_int_t     NRHS = 1;   // Number of Right Hand Sides, a.k.a.

     /* Initialize */
     magma_queue_t queue;
     magma_device_t device;
     magma_int_t num = 0;
     magma_int_t     status;
     magma_int_t ldda = md;

     status = magma_init();
     if (status != MAGMA_SUCCESS)
     {
         printf("MAGMA magma_init FAILED with %d\n", status);
     }

     status = magma_get_devices( &device, 1, &num );
     if ( status != MAGMA_SUCCESS or num < 1 ) {
         fprintf( stderr, "magma_get_devices failed: %d\n", (int) status );
         exit(-1);
     }
     status = magma_queue_create( device, &queue );
     if ( status != MAGMA_SUCCESS ) {
         fprintf( stderr, "magma_queue_create failed: %d\n", (int) status );
         exit(-1);
     }


     // MAGMA way
     TESTING_MALLOC_HOST( h_D, double, md * md   );
     TESTING_MALLOC_HOST( h_sv, double, ldda * NRHS );
     TESTING_MALLOC_HOST(IPIV, magma_int_t, md);

     TESTING_MALLOC_DEV(d_D,    double, md * ldda);
     TESTING_MALLOC_DEV(d_sv,   double, ldda * NRHS);

     // testing malloc
     //printf("I malloced h_D upper level %d", md);



     //#pragma omp parallel for private(l, i, irr, A, j, ir, k) num_threads(nthreads)// par1
     for (k = 0; k < NUM_AIRFOILS; k++)
     {
         if (k == 0) ir = 0; // nqkakav broqch
         else ir = suma(N, k);

#pragma omp parallel for private(l, i, irr, A, j) num_threads(nthreads)// par2
         for (j = 0; j < (N-1); j++)
         {
             A = vort(N, points.x, points.y, points);
             // testing
             // printf("I malloced A from vort %d %d\n", k, j);

             //#pragma omp parallel for private(l, i, irr) num_threads(nthreads)// par3
             for (l = 0; l < NUM_AIRFOILS ; l++)
             {
                 // if (k == 0 && j == 0 && l == 0 ) printf("NUM threads = %d\n", omp_get_num_threads());
                 if (l == 0) irr = 0;
                 else irr = suma(N, l) - l;
                 // in this for we calculate lots of stuff but without 1 and last point in every airfoil

                 // #pragma omp parallel for private(i) num_threads(nthreads)// par4
                 for (i = 0; i < N-2; i++)
                 {

                     //if (k == 0 && j == 0 && l == 0 && i == 0) printf("NUM threads = %d\n", omp_get_num_threads());
                     h_D [ (irr + i - l)    * md + (ir + j - k) ] =
                         - A[irr + i + l +1]/(8*M_PI);


                 }
             }
             // Freeing A so we could malloc it again
             free(A);

             // Adding values at the end of the matrix
             for (i = 0; i < NUM_AIRFOILS; i++)
             {
                 if (i == k)
                     h_D[ (md - NUM_AIRFOILS + i) * md + (ir +j -k) ] = -1;
                 //h_D[ir +j -k][md - NUM_AIRFOILS + i] = -1;
                 else
                     h_D[ (md - NUM_AIRFOILS + i) * md + (ir +j -k) ] = 0;
                 //h_D[ir +j -k][md - NUM_AIRFOILS + i] = 0;
             }
             h_sv[ir + j - k] = points.y;
         }
     }

     // end of parallel section

     ////////////    T   // I   // M   // E   //////////////
     // t   /   e   /   s   /   t   /   i   /   n   /   g   //
     // time after finding h_D
     gettimeofday( &direct_init_t, (struct timezone*)0);

     //////////////////////////////////////////////////////////

     // Set data to be used by MIC
     magma_dsetmatrix( md, md,   h_D, 0, md*ldda, d_D, 0, ldda, queue );
     magma_dsetmatrix( md, NRHS, h_sv, 0, md, d_sv, 0, ldda, queue );

     // ADD magma dgesv
     magma_dgesv_mic( md, NRHS, h_D, 0, ldda, IPIV, h_sv, 0, ldda, &info, queue );
     if (info != 0) {
         printf("magma_dgesv returned error %lld: %s.\n",
                (long long) info, magma_strerror( info ));
     }



     // test if we had a solution PLASMA
     //    if (INFO < 0)
     //    {
     //        printf("-- Error %d in dgesv", INFO);
     //        exit(EXIT_FAILURE);
     //    }

     //Deallocate L and IPIV /
     //    free(L); // this was PLASMA thing TODO remove me
     free(IPIV);
     // Plasma Finalize /
     // INFO = PLASMA_Finalize();

     // TODO Add MAGMA Finalize
     //
     // MAGMA Finalize

     ////////////    T   // I   // M   // E   //////////////
     // t   /   e   /   s   /   t   /   i   /   n   /   g   //
     // time after solve
     gettimeofday( &direct_solve_t, (struct timezone*)0);
     //////////////////////////////////////////////////////////

     // Prehvarlqne na st-stite ot h_sv v relanoto GA
     // (ako izobshto q ima tazi chast nakraq)

     int j0 = 0;
     for (i = 0; i < NUM_AIRFOILS; i++)
     {
         points[0].ga = 0.0;
         for (j = 1; j < N; j++)
         {
             points.ga = h_sv[j+j0-1];
         }
         j0 += N-1;
     }

     ////////////    T   // I   // M   // E   //////////////
     // t   /   e   /   s   /   t   /   i   /   n   /   g   //
     // time of ga allocation
     gettimeofday( &direct_alocate_t, (struct timezone*)0);
     //////////////////////////////////////////////////////////

//    magma_free_cpu(h_D);
//    magma_free_cpu(h_sv);
//    magma_free_cpu(IPIV);
     /* Shutdown */
     magma_queue_destroy( queue );
     magma_finalize();
     if (status != MAGMA_SUCCESS)
     {
         printf("MAGMA magma_finalize with %d\n", status);
     }
}

EDIT: Fixed typo in code. There was some vim text left in it.

Dimitar_S_1 · ‎10-02-2017

Problem solved.

A really dumb mistake actually. In:

magma_dgesv_mic( md, NRHS, h_D, 0, ldda, IPIV, h_sv, 0, ldda, &info, queue );

h_D and h_sv are the matrix and the B part on the CPU. I have to use d_D and d_sv instead. So the above becomes:

magma_dgesv_mic( md, NRHS, d_D, 0, ldda, IPIV, d_sv, 0, ldda, &info, queue );