topic Hi Craig, in Intel® oneAPI DPC++/C++ Compiler

runtime error: ZE_RESULT_ERROR_INVALID_ENUMERATION

CFR — Sun, 07 Jun 2020 19:04:22 GMT

I guess my two major questions are:

Why the random runtime error.
Why can't I catch the const char * exception and see what it says.

Environment:

Lenovo T570
Ubuntu 18.04.4 LTS
5.3.0-53-generic
g++ (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0

Compile Line:

dpcpp -g -o bug2 bug2.cpp -lOpenCL -lsycl

Sometimes when I execute, I get the following:

  user@t570:~/OneAPI/Code$ ./bug2
  Device: name: Intel(R) Gen9
  vendor: Intel
  Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
  pi_throw: L0 Error
  Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
  pi_throw: L0 Error
  Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
  pi_throw: L0 Error
  terminate called after throwing an instance of 'char const*'
  Aborted (core dumped)

Sometimes I get what I expect:

  user@t570:~/OneAPI/Code$ ./bug2
  Device: name: Intel(R) Gen9
  vendor: Intel
   0    1    2    3    4    5    6
  10   11   12   13   14   15   16
  20   21   22   23   24   25   26
  30   31   32   33   34   35   36

According to GDB it looks to be in the buffer code, but I'm not exactly sure what my part is in the problem:

#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007ffff6e7d801 in __GI_abort () at abort.c:79
#2 0x00007ffff7870957 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3 0x00007ffff7876ae6 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00007ffff7876b21 in std::terminate() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#5 0x0000000000404c4f in __clang_call_terminate ()
at /local/opt/intel/inteloneapi/compiler/latest/linux/bin/../include/sycl/CL/sycl/buffer.hpp:218
#6 0x0000000000409656 in cl::sycl::detail::buffer_impl::~buffer_impl (this=0x8fffa0)
at /local/opt/intel/inteloneapi/compiler/latest/linux/bin/../include/sycl/CL/sycl/detail/buffer_impl.hpp:111
#7 0x00000000004095f9 in __gnu_cxx::new_allocator<cl::sycl::detail::buffer_impl>::destroy<cl::sycl::detail::buffer_impl> (
this=0x8fffa0, __p=0x8fffa0) at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/ext/new_allocator.h:140
#8 0x0000000000409590 in std::allocator_traits<std::allocator<cl::sycl::detail::buffer_impl> >::destroy<cl::sycl::detail::buffer_impl> (__a=..., __p=0x8fffa0) at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/alloc_traits.h:487
#9 0x000000000040694c in std::_Sp_counted_ptr_inplace<cl::sycl::detail::buffer_impl, std::allocator<cl::sycl::detail::buffer_impl>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x8fff90)
at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/shared_ptr_base.h:535
#10 0x000000000040505c in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x8fff90)
at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/shared_ptr_base.h:154
#11 0x000000000040500a in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count (this=0x7fffffffbe70)
at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/shared_ptr_base.h:684
#12 0x000000000040aade in std::__shared_ptr<cl::sycl::detail::buffer_impl, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (
this=0x7fffffffbe68) at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/shared_ptr_base.h:1123
#13 0x00000000004054a8 in std::shared_ptr<cl::sycl::detail::buffer_impl>::~shared_ptr (this=0x7fffffffbe68)
at /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/shared_ptr.h:93
#14 0x0000000000404c35 in cl::sycl::buffer<unsigned int, 1, cl::sycl::detail::aligned_allocator<char>, void>::~buffer (
this=0x7fffffffbe68) at /local/opt/intel/inteloneapi/compiler/latest/linux/bin/../include/sycl/CL/sycl/buffer.hpp:218
#15 0x0000000000403380 in main (argc=1, argv=0x7fffffffc0d8) at bug2.cpp:54

Here's a pared down version that generated the above:

#define XN 7
#define Nproc 4

#include <cstdio>
#include <CL/sycl.hpp>
namespace sycl = cl::sycl;

int
main(int argc, char *argv[])
{
  unsigned int x[Nproc][XN];

  //sycl::device dev = sycl::default_selector().select_device();
  //sycl::device dev = sycl::host_selector().select_device();
  sycl::device dev = sycl::gpu_selector().select_device();
  //sycl::device dev = sycl::cpu_selector().select_device();
  std::cout << "Device: " 
      << "name: " << dev.get_info<sycl::info::device::name>() << std::endl
      << "vendor: " << dev.get_info<sycl::info::device::vendor>() << std::endl;

  sycl::queue q(dev);

  try {
    sycl::buffer<unsigned int, 1> xbuffer((unsigned int *)x, sycl::range<1> {Nproc*XN});
    q.submit([&](sycl::handler& cgh) {
        auto xaccessor = xbuffer.get_access<sycl::access::mode::discard_write, sycl::access::target::global_buffer>(cgh);
        cgh.parallel_for<class k0>(
            sycl::range<1> {Nproc}, 
            [=] (sycl::item<1> item) {
                int idx = item.get_linear_id();
                unsigned int (*xptr)[XN] = (unsigned int (*)[XN])xaccessor.get_pointer().get();
                for (int i=0; i<XN; i++) xptr[idx] = 10*idx+i;
                }
            );
        }
        ); 
    q.wait();
  } catch (sycl::runtime_error const& e) {
    std::cout << "SYCL runtime exception" << std::endl;
    std::cout << e.what() << std::endl;
  } catch (sycl::device_error const & e) {
    std::cout << "SYCL device exception" << std::endl;
    std::cout << e.what() << std::endl;
  } catch (const char * str) {
    std::cout << "msg: " << str << std::endl;
  } catch (...) {
    std::cout << "SYCL unknown exception" << std::endl;
  }
  for (int p=0; p<Nproc; p++) {
    for (int i=0; i<XN; i++) printf("%2d   ", x);
    printf("\n");
  }
}

Hi Craig,

RahulV_intel — Mon, 08 Jun 2020 09:29:37 GMT

Hi Craig,

Could you try setting the Sycl Back-End to OpenCL and let us know if the inconsistency still persists at runtime?

Follow these steps for compiling and running your code(with OpenCL back-end):

dpcpp -g -o bug2 bug2.cpp
SYCL_BE=PI_OPENCL ./bug2

Regards,

Rahul

This seems to be the only

CFR — Mon, 08 Jun 2020 14:37:13 GMT

This seems to be the only output now... I do not see it any runs with the ZE_RESULT_ERROR_INVALID_ENUMERATION message (but maybe that's expected).

user@t570:~/OneAPI/Code$ SYCL_BE=PI_OPENCL ./bug2
Device: name: Intel(R) Gen9 HD Graphics NEO
vendor: Intel(R) Corporation
DIBlockByRefStruct on DICompositeType is no longer supported
!239 = !DICompositeType(tag: DW_TAG_structure_type, name: "__private_class", scope: !211, file: !212, line: 34, flags: DIFlagReservedBit4, elements: !137)
DIBlockByRefStruct on DICompositeType is no longer supported
!766 = !DICompositeType(tag: DW_TAG_structure_type, name: "multi_ptr<const unsigned int, cl::sycl::access::address_space::global_space>", scope: !145, file: !691, line: 25, flags: DIFlagReservedBit4, elements: !137)
 0    1    2    3    4    5    6   
10   11   12   13   14   15   16   
20   21   22   23   24   25   26   
30   31   32   33   34   35   36

(I'm in the process of trying to add and async_handler to my program to see if it's more informative)

Hi,

RahulV_intel — Tue, 09 Jun 2020 04:58:22 GMT

Hi,

Did you notice any runtime inconsistency using OpenCL banckend with multiple runs?

Also, let us know if there is any discrepancy in output during these runs.

Try out the asyc_handler with default L0 backend as well and check if it catches any error when the runtime fails(in previous case).

Thanks,

Rahul

I ran "./bug2" 10 times. Out

CFR — Tue, 09 Jun 2020 21:25:41 GMT

I ran "./bug2" 10 times. Out of the 10

4 run correctly:

Device: name: Intel(R) Gen9
vendor: Intel
 0    1    2    3    4    5    6   
10   11   12   13   14   15   16   
20   21   22   23   24   25   26   
30   31   32   33   34   35   36

6 result in error:

Device: name: Intel(R) Gen9
vendor: Intel
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
terminate called after throwing an instance of 'char const*'

I ran "SYCL_BE=PI_OPENCL ./bug2" 10 times and all 10 runs resulted in the exact same output:

Device: name: Intel(R) Gen9 HD Graphics NEO
vendor: Intel(R) Corporation
DIBlockByRefStruct on DICompositeType is no longer supported
!239 = !DICompositeType(tag: DW_TAG_structure_type, name: "__private_class", scope: !211, file: !212, line: 34, flags: DIFlagReservedBit4, elements: !137)
DIBlockByRefStruct on DICompositeType is no longer supported
!766 = !DICompositeType(tag: DW_TAG_structure_type, name: "multi_ptr<const unsigned int, cl::sycl::access::address_space::global_space>", scope: !145, file: !691, line: 25, flags: DIFlagReservedBit4, elements: !137)
 0    1    2    3    4    5    6   
10   11   12   13   14   15   16   
20   21   22   23   24   25   26   
30   31   32   33   34   35   36

I attempted to implement an async handler as follows:

  /** create async exception handler per codeplay example **/
  auto exception_handler = [] (sycl::exception_list exceptions) {
    for (std::exception_ptr const& e : exceptions) {
      try {
        std::rethrow_exception(e);
      } catch(sycl::exception const& e) {
        std::cout << "Caught asynchronous SYCL exception:\n" << e.what() << std::endl;
      }
    }
  };

  /** setup a queue **/
  sycl::queue q(dev, exception_handler);

  try {
    sycl::buffer<unsigned int, 1> xbuffer((unsigned int *)x, sycl::range<1> {Nproc*XN});
    q.submit([&](sycl::handler& cgh) {
        auto xaccessor = xbuffer.get_access<sycl::access::mode::discard_write, sycl::access::target::global_buffer>(cgh);
        cgh.parallel_for<class k0>(
            sycl::range<1> {Nproc},
            [=] (sycl::item<1> item) {
                int idx = item.get_linear_id();
                unsigned int (*xptr)[XN] = (unsigned int (*)[XN])xaccessor.get_pointer().get();
                for (int i=0; i<XN; i++) xptr[idx] = 10*idx+i;
                }
            );
        }
        );
    q.wait_and_throw();
  } catch (sycl::runtime_error const& e) {
    std::cout << "SYCL runtime exception" << std::endl;
    std::cout << e.what() << std::endl;
  } catch (sycl::device_error const & e) {
    std::cout << "SYCL device exception" << std::endl;
    std::cout << e.what() << std::endl;
  } catch (const char * str) {
    std::cout << "msg: " << str << std::endl;
  } catch (...) {
    std::cout << "SYCL unknown exception" << std::endl;
  }

But it didn't produce any more information.

Some other things about this...

I have other, similar, codes that exhibit the same problem
The error does _not_ occur for the cpu_selector, only for the gpu_selector

Edit 1: After poking through some source it turns out that setting ZE_DEBUG environment variable will trace the internal calls. Not sure whether it sheds too much new information but maybe...

ZE ---> zeInit(ZE_INIT_FLAG_NONE)
ZE ---> piPlatformsGet
ZE ---> zeInit(ZE_INIT_FLAG_NONE)
ZE ---> piPlatformsGet
ZE ---> zeDriverGet(&ze_driver_count, nullptr)
ZE ---> zeDriverGet(&ze_driver_count, &ze_driver)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, nullptr)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, ze_devices)
ZE ---> zeDriverGet(&ze_driver_count, nullptr)
ZE ---> zeDriverGet(&ze_driver_count, &ze_driver)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, nullptr)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, ze_devices)
ZE ---> zeCommandListCreateImmediate(ze_device, &ze_command_queue_desc, &L0PiDevice->L0CommandListInit)
ZE ---> zeDeviceGetProperties( ze_device, &L0PiDevice->L0DeviceProperties)
ZE ---> zeDeviceGetComputeProperties( ze_device, &L0PiDevice->L0DeviceComputeProperties)
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDriverGet(&ze_driver_count, nullptr)
ZE ---> zeDriverGet(&ze_driver_count, &ze_driver)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, nullptr)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, ze_devices)
ZE ---> zeDriverGet(&ze_driver_count, nullptr)
ZE ---> zeDriverGet(&ze_driver_count, &ze_driver)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, nullptr)
ZE ---> zeDeviceGet(ze_driver, &ze_device_count, ze_devices)
ZE ---> zeCommandListCreateImmediate(ze_device, &ze_command_queue_desc, &L0PiDevice->L0CommandListInit)
ZE ---> zeDeviceGetProperties( ze_device, &L0PiDevice->L0DeviceProperties)
ZE ---> zeDeviceGetComputeProperties( ze_device, &L0PiDevice->L0DeviceComputeProperties)
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
Device: name: Intel(R) Gen9
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, nullptr )
ZE ---> zeDeviceGetMemoryProperties( ze_device, &ze_avail_mem_count, ze_device_memory_properties )
ZE ---> zeDeviceGetImageProperties( ze_device, &ze_device_image_properties )
ZE ---> zeDeviceGetKernelProperties( ze_device, &ze_device_kernel_properties )
ZE ---> zeDeviceGetCacheProperties( ze_device, &ze_device_cache_properties )
vendor: Intel
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeDriverGetProperties(ze_driver_global, &ze_driver_properties)
==========================
SYCL over Level-Zero 0.8
==========================
ZE ---> zeCommandQueueCreate( ze_device, &ze_command_queue_desc, &ze_command_queue)
ZE ---> zeDriverAllocDeviceMem( ze_driver_global, &ze_desc, size, 1, ze_device, &ptr)
ZE ---> zeCommandListAppendMemoryCopy(context->Device->L0CommandListInit, ptr, host_ptr, size, nullptr)
ZE ---> zeModuleCreate( ze_device, &ze_module_desc, &ze_module, 0)
ZE ---> zeKernelCreate( pi_cast<ze_module_handle_t>(program->L0Module), &ze_kernel_desc, &ze_kernel)
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), sizeof(void *), pi_cast<const void*>(&(*arg_value)->L0Mem))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_SHARED_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_DEVICE_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_HOST_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSuggestGroupSize(kernel->L0Kernel, global_work_size[0], global_work_size[1], global_work_size[2], &wg[0], &wg[1], &wg[2])
ZE ---> zeKernelSetGroupSize(kernel->L0Kernel, wg[0], wg[1], wg[2])
ZE ---> zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), sizeof(void *), pi_cast<const void*>(&(*arg_value)->L0Mem))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_SHARED_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_DEVICE_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_HOST_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSuggestGroupSize(kernel->L0Kernel, global_work_size[0], global_work_size[1], global_work_size[2], &wg[0], &wg[1], &wg[2])
ZE ---> zeKernelSetGroupSize(kernel->L0Kernel, wg[0], wg[1], wg[2])
ZE ---> zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), sizeof(void *), pi_cast<const void*>(&(*arg_value)->L0Mem))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(kernel->L0Kernel), pi_cast<uint32_t>(arg_index), pi_cast<size_t>(arg_size), pi_cast<const void*>(arg_value))
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_SHARED_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_DEVICE_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSetAttribute( kernel->L0Kernel, ZE_KERNEL_ATTR_INDIRECT_HOST_ACCESS, sizeof(bool), &ze_indirect_value)
ZE ---> zeKernelSuggestGroupSize(kernel->L0Kernel, global_work_size[0], global_work_size[1], global_work_size[2], &wg[0], &wg[1], &wg[2])
ZE ---> zeKernelSetGroupSize(kernel->L0Kernel, wg[0], wg[1], wg[2])
ZE ---> zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
terminate called after throwing an instance of 'char const*'
Aborted (core dumped)

Hi,

RahulV_intel — Wed, 10 Jun 2020 06:06:00 GMT

Hi,

Thanks for providing detailed info. It looks like a bug with the L0 backend from the log.

I have other, similar, codes that exhibit the same problem
The error does _not_ occur for the cpu_selector, only for the gpu_selector

This issue is with L0 backend or OpenCL backend?

A point to note here is that the default Backend for GPU(or any accelerator for that matter) would be L0, unless you change it explicitly at rutime (using SYCL_BE variable). That should answer your question as to why it doesn't fail when you use cpu_selector (Since default BE for CPU device is OpenCL).

After poking through some source it turns out that setting ZE_DEBUG environment variable will trace the internal calls.

We have a similar environment variable SYCL_PI_TRACE=2 (The integer here indicates the volume of debug log) to trace the internal calls.

I will escalate the L0 backend runtime inconsistency issue to the concerned team.

--Rahul

Not sure I know enough to

CFR — Wed, 10 Jun 2020 18:45:53 GMT

Not sure I know enough to correctly answer, but using ```sycl::device dev = sycl::cpu_selector().select_device();``` never produces an error. Using ```sycl::device dev = sycl::gpu_selector().select_device();``` does produce errors. If you're saying "cpu_selector" and using "SYCL_BE=PI_OPENCL" are equivalent, then that would make sense.

I tried running with SYCL_PI_TRACE=2 and I get close to 1000 lines of output, but it does not ever produce the error (tried more than 100 executions). For yucks I tried SYCL_PI_TRACE=1 and it's pretty much back to my original case with random executions producing errors:

user@t570:~/OneAPI/Code$ SYCL_PI_TRACE=1 ./bug2
SYCL_PI_TRACE[1]: Plugin found and successfully loaded: libpi_opencl.so
SYCL_PI_TRACE[1]: Plugin found and successfully loaded: libpi_level0.so
SYCL_PI_TRACE[1]: select_device(): ->
SYCL_PI_TRACE[1]:   platform: Level-Zero 0.8
SYCL_PI_TRACE[1]:   device: Intel(R) Gen9
Device: name: Intel(R) Gen9
vendor: Intel
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
Error (ZE_RESULT_ERROR_INVALID_ENUMERATION) in zeCommandListCreate( L0Device, &ze_command_list_desc, &ze_command_list)
pi_throw: L0 Error
terminate called after throwing an instance of 'char const*'
Aborted (core dumped)

It looks like I never mentioned the OneAPI version I'm using:

user@t570:~/OneAPI/Code$ dpcpp --version
Intel(R) oneAPI DPC++ Compiler 2021.1-beta06 (2020.4.0.0415)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /local/opt/intel/inteloneapi/compiler/latest/linux/bin

If you need any additional information on my environment let me know. I'd also be happy to run an instrumented libpi_level0.so if it would help uncover just what zeCommandListCreate is unhappy about.

Hi,

RahulV_intel — Thu, 11 Jun 2020 05:47:00 GMT

Hi,

Not sure I know enough to correctly answer, but using ```sycl::device dev = sycl::cpu_selector().select_device();``` never produces an error. Using ```sycl::device dev = sycl::gpu_selector().select_device();``` does produce errors. If you're saying "cpu_selector" and using "SYCL_BE=PI_OPENCL" are equivalent, then that would make sense.

Not exactly! DPC++ runtime offers multiple plugin interfaces(Backends) for the device execution to take place.

The DPC++ Runtime Plugin Interface (PI) is an interface layer between the device-agnostic part of DPC++ runtime and the device-specific runtime layers which control execution on devices. It employs the “plugin” mechanism to bind to the device specific runtime layers like OpenCL/Level0(L0).

When you select gpu_selector(), it defaults to Level0 backend(BE) as per the current implementation.

SYCL_BE environment variable alters this default behavior.

SYCL_BE=PI_OPENCL ./bug2

What the above statement essentially does is that, it asks DPC++ runtime to change the default behavior from Level0(L0) to OpenCL.

Previously, when you had not set this environment variable, it was running on Level0 default backend(On GPU). That is the reason I had asked you to change the default BE to OpenCL and check if you are still facing the same error.

Coming back to cpu_selector(), the current implementation by default sets OpenCL backend PI, in case of a CPU device. Even though you did not set the SYCL_BE variable explicitly in this case, it was by default using OpenCL BE. That is the reason you did not observe any issues with cpu_selector() (since its running on OpenCL BE by default).

The reason that I had asked you to check with OpenCL BE(for gpu_selector()) is to see whether you observe similar behavior there too. As per your claims, you did not observe any runtime inconsistencies with OpenCL. Therefore, I had suspected it to be an L0 bug.

Hope this helps.

--Rahul

appears to be fixed in beta07

CFR — Mon, 22 Jun 2020 00:29:50 GMT

appears to be fixed by beta07

I upgraded to beta07 and this problem seems to have gone away. Not sure if it was something in the OneAPI distribution that got fixed or whether I "fixed" something when I updated the software, but... I will note in the following that platform is now 0.91 vs. 0.8 before.

user@t570:~/OneAPI/Code$ SYCL_PI_TRACE=1 ./bug2
SYCL_PI_TRACE[basic]: Plugin found and successfully loaded: libpi_opencl.so
SYCL_PI_TRACE[basic]: Plugin found and successfully loaded: libpi_level0.so
SYCL_PI_TRACE[all]: Selected device ->
SYCL_PI_TRACE[all]:   platform: 0.91
SYCL_PI_TRACE[all]:   device: Intel(R) Gen9
Device: name: Intel(R) Gen9
vendor: Intel(R) Corporation
 0    1    2    3    4    5    6   
10   11   12   13   14   15   16   
20   21   22   23   24   25   26   
30   31   32   33   34   35   36

Though ZE_DEBUG seems to indicate version went from 0.8 -> 0.0.8. *shrug*

user@t570:~/OneAPI/Code$ ZE_DEBUG=1 ./bug2
ZE ---> zeInit
ZE ---> zeInit
ZE ---> zeDriverGet(&ZeDriverCount, nullptr)
ZE ---> zeDriverGet(&ZeDriverCount, &ZeDriver)
ZE ---> zeDriverGetProperties(ZeDriver, &ZeDriverProperties)
ZE ---> zeDriverGetApiVersion(ZeDriver, &ZeApiVersion)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, nullptr)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, ZeDevices)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, nullptr)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, ZeDevices)
ZE ---> zeCommandListCreateImmediate(ZeDevice, &ze_command_queue_desc, &ZeCommandListInit)
ZE ---> zeDeviceGetProperties(ZeDevice, &ZeDeviceProperties)
ZE ---> zeDeviceGetComputeProperties(ZeDevice, &ZeDeviceComputeProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, nullptr)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, ZeDevices)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, nullptr)
ZE ---> zeDeviceGet(ZeDriver, &ZeDeviceCount, ZeDevices)
ZE ---> zeCommandListCreateImmediate(ZeDevice, &ze_command_queue_desc, &ZeCommandListInit)
ZE ---> zeDeviceGetProperties(ZeDevice, &ZeDeviceProperties)
ZE ---> zeDeviceGetComputeProperties(ZeDevice, &ZeDeviceComputeProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
Device: name: Intel(R) Gen9
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, nullptr)
ZE ---> zeDeviceGetMemoryProperties(ZeDevice, &ZeAvailMemCount, ZeDeviceMemoryProperties)
ZE ---> zeDeviceGetImageProperties(ZeDevice, &ZeDeviceImageProperties)
ZE ---> zeDeviceGetKernelProperties(ZeDevice, &ZeDeviceKernelProperties)
ZE ---> zeDeviceGetCacheProperties(ZeDevice, &ZeDeviceCacheProperties)
vendor: Intel(R) Corporation
==========================
SYCL over Level-Zero 0.0.8
==========================
==========================
SYCL over Level-Zero 0.0.8
==========================
ZE ---> zeCommandQueueCreate(ZeDevice, &ZeCommandQueueDesc, &ZeCommandQueue)
ZE ---> zeDriverAllocDeviceMem(Context->Device->Platform->ZeDriver, &ZeDesc, Size, 1, ZeDevice, &Ptr)
ZE ---> zeCommandListAppendMemoryCopy(Context->Device->ZeCommandListInit, Ptr, HostPtr, Size, nullptr)
ZE ---> zeModuleCreate(ZeDevice, &Program->ZeModuleDesc, &Program->ZeModule, &Program->ZeBuildLog)
ZE ---> zeKernelCreate(pi_cast<ze_module_handle_t>(Program->ZeModule), &ZeKernelDesc, &ZeKernel)
ZE ---> zeKernelSetArgumentValue(pi_cast<ze_kernel_handle_t>(Kernel->ZeKernel), pi_cast<uint32_t>(ArgIndex), sizeof(void *), (*ArgValue)->getZeHandlePtr())
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(Kernel->ZeKernel), pi_cast<uint32_t>(ArgIndex), pi_cast<size_t>(ArgSize), pi_cast<const void *>(ArgValue))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(Kernel->ZeKernel), pi_cast<uint32_t>(ArgIndex), pi_cast<size_t>(ArgSize), pi_cast<const void *>(ArgValue))
ZE ---> zeKernelSetArgumentValue( pi_cast<ze_kernel_handle_t>(Kernel->ZeKernel), pi_cast<uint32_t>(ArgIndex), pi_cast<size_t>(ArgSize), pi_cast<const void *>(ArgValue))
ZE ---> zeKernelSetAttribute(Kernel->ZeKernel, ZE_KERNEL_ATTR_INDIRECT_SHARED_ACCESS, sizeof(bool), &ZeIndirectValue)
ZE ---> zeKernelSetAttribute(Kernel->ZeKernel, ZE_KERNEL_ATTR_INDIRECT_DEVICE_ACCESS, sizeof(bool), &ZeIndirectValue)
ZE ---> zeKernelSetAttribute(Kernel->ZeKernel, ZE_KERNEL_ATTR_INDIRECT_HOST_ACCESS, sizeof(bool), &ZeIndirectValue)
ZE ---> zeKernelSuggestGroupSize(Kernel->ZeKernel, GlobalWorkSize[0], GlobalWorkSize[1], GlobalWorkSize[2], &WG[0], &WG[1], &WG[2])
ZE ---> zeKernelSetGroupSize(Kernel->ZeKernel, WG[0], WG[1], WG[2])
ZE ---> zeCommandListCreate(ZeDevice, &ZeCommandListDesc, ZeCommandList)
ZE ---> Context->getFreeSlotInExistingOrNewPool(ZeEventPool, Index)
ZE ---> zeEventCreate(ZeEventPool, &ZeEventDesc, &ZeEvent)
ZE ---> zeCommandListAppendLaunchKernel( ZeCommandList, Kernel->ZeKernel, &ZeThreadGroupDimensions, ZeEvent, NumEventsInWaitList, ZeEventWaitList)
calling zeCommandListAppendLaunchKernel() with  ZeEvent 33396a0
  NumEventsInWaitList 0:
ZE ---> zeCommandListClose(ZeCommandList)
ZE ---> zeCommandQueueExecuteCommandLists(ZeCommandQueue, 1, &ZeCommandList, nullptr)
ZeEvent = 33396a0
ZE ---> zeEventHostSynchronize(ZeEvent, 100000)
ZE ---> ZeResult
ZE ---> zeCommandListDestroy(EventList->ZeCommandList)
ZE ---> zeCommandListCreate(ZeDevice, &ZeCommandListDesc, ZeCommandList)
ZE ---> Context->getFreeSlotInExistingOrNewPool(ZeEventPool, Index)
ZE ---> zeEventCreate(ZeEventPool, &ZeEventDesc, &ZeEvent)
ZE ---> zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList, ZeEventWaitList)
ZE ---> zeCommandListAppendMemoryCopy(ZeCommandList, Dst, Src, Size, ZeEvent)
ZE ---> zeCommandListClose(ZeCommandList)
ZE ---> zeCommandQueueExecuteCommandLists(ZeCommandQueue, 1, &ZeCommandList, nullptr)
calling zeCommandListAppendMemoryCopy() with
  xe_event 32d04f0
  NumEventsInWaitList 1: 33396a0
ZeEvent = 32d04f0
ZE ---> zeEventHostSynchronize(ZeEvent, 100000)
ZE ---> ZeResult
ZE ---> zeCommandListDestroy(EventList->ZeCommandList)
ZeEvent = 32d04f0
ZE ---> zeEventHostSynchronize(ZeEvent, 100000)
ZE ---> ZeResult
ZeEvent = 33396a0
ZE ---> zeEventHostSynchronize(ZeEvent, 100000)
ZE ---> ZeResult
ZE ---> zeDriverFreeMem(Mem->Platform->ZeDriver, Mem->getZeHandle())
ZE ---> zeEventDestroy(Event->ZeEvent)
ZE ---> Context->decrementAliveEventsInPool(Event->ZeEventPool)
ZE ---> zeEventDestroy(Event->ZeEvent)
ZE ---> Context->decrementAliveEventsInPool(Event->ZeEventPool)
 0    1    2    3    4    5    6   
10   11   12   13   14   15   16   
20   21   22   23   24   25   26   
30   31   32   33   34   35   36   
ZE ---> zeCommandQueueDestroy(Queue->ZeCommandQueue)

Not really sure what the story is but if you're reading this because you get the ZE_RESULT_ERROR_INVALID_ENUMERATION error try updating to beta07. Worked for me.

Re:runtime error: ZE_RESULT_ERROR_INVALID_ENUMERATION

Subarnarek_G_Intel — Mon, 03 Aug 2020 11:59:42 GMT

This issue has been resolved and we will no longer respond to this thread. If you require additional assistance from Intel, please start a new thread. Any further interaction in this thread will be considered community only.