Get nan error when use SSD-MobileNet via C++

[idata]

I use the SSD-MobileNet model in the ncappzoo, it's ok via python. But get nan error via C++. I use the same graph file and the same test image. The key code is as follows: ... for(int i = 0; i < resize_height; i++) { for(int j = 0; j < resize_width; j++) { for(int k = 0; k < channel; k++) { img_processed_data_float[i*resize_width*channel + j*channel + k] = (float(img_processed.at<cv::Vec3b>(i,j)[k]) - mean ) * scale; } } } half * img_processed_data_half = new half[img_processed_length]; floattofp16((unsigned char *)img_processed_data_half, img_processed_data_float, img_processed_length); // start the inference with mvncLoadTensor() mvncStatus retCode = mvncLoadTensor(graph_handle_, img_processed_data_half, img_processed_length, NULL); // get the detect results void* result_data_half; void* userParam; unsigned int result_data_length; retCode = mvncGetResult(graph_handle_, &result_data_half, &result_data_length, &userParam); // post process detect results result_data_length = result_data_length / sizeof(half); float * result_data_float = new float[result_data_length]; fp16tofloat(result_data_float, (unsigned char *)result_data_half, result_data_length); the detection results are: box 0: 20, 0.191895, -0.119263, 55776, 0.000798702, 0.220093, -0.00656509 box 1: 0, 1, nan, nan, nan, nan, nan box 2: 0, 2, nan, nan, nan, nan, nan box 3: 0, 3, nan, nan, nan, nan, nan box 4: 0, 4, nan, nan, nan, nan, nan box 5: 0, 5, nan, nan, nan, nan, nan box 6: 0, 6, nan, nan, nan, nan, nan box 7: 0, 7, nan, nan, nan, nan, nan box 8: 0, 8, nan, nan, nan, nan, nan box 9: 0, 9, nan, nan, nan, nan, nan box 10: 0, 10, nan, nan, nan, nan, nan box 11: 0, 11, nan, nan, nan, nan, nan box 12: 0, 12, nan, nan, nan, nan, nan box 13: 0, 13, nan, nan, nan, nan, nan box 14: 0, 14, nan, nan, nan, nan, nan ...

[idata]

@songyoff Can you provide your entire app code? I would like to reproduce your code on my machine setup. Thanks.

[idata]

@Tome_at_Intel Thanks for your reply. I had found the bug. The thing may be that the input data for ncs is stored in col-major order, however, I stored the data in row-major order.

[idata]

@songyoff Glad you were able to solve the issue.

[idata]

Hi @songyoff and @Tome_at_Intel , I am facing your same issue trying to implement the live object detector in C++ for NCSDK v2. Could you please look into my code or share with me a working code? I have tried row major, col major, changing tensor datatype, rgb and bgr conversion and everything I could think of. I am new to this and would be grateful if you could help me out.

//Skipped initial few codes as i think the problem is somewhere here bool preprocess_image(const cv::Mat& src_image_mat, cv::Mat& preprocessed_image_mat) { // find ratio of to adjust width and height by to make them fit in network image width and height double width_ratio = (double)NETWORK_IMAGE_WIDTH / (double)src_image_mat.cols; double height_ratio = (double)NETWORK_IMAGE_HEIGHT / (double)src_image_mat.rows; // the largest ratio is the one to use for scaling both height and width. double largest_ratio = (width_ratio > height_ratio) ? width_ratio : height_ratio; cv::resize(src_image_mat, preprocessed_image_mat, cv::Size(), largest_ratio, largest_ratio, CV_INTER_AREA); int mid_row = preprocessed_image_mat.rows / 2.0; int mid_col = preprocessed_image_mat.cols / 2.0; int x_start = mid_col - (NETWORK_IMAGE_WIDTH / 2); int y_start = mid_row - (NETWORK_IMAGE_HEIGHT / 2); cv::Rect roi(x_start, y_start, NETWORK_IMAGE_WIDTH, NETWORK_IMAGE_HEIGHT); preprocessed_image_mat = preprocessed_image_mat(roi); //RGB to BRG //cvtColor(preprocessed_image_mat, preprocessed_image_mat, COLOR_RGB2BGR); //To know the preprocessed image //imshow(WINDOW_NAME, preprocessed_image_mat); return true; } std::shared_ptr<list<networkResults>> getInferenceResults(cv::Mat inputMat, struct ncGraphHandle_t* graphHandle, struct ncFifoHandle_t* fifoIn, struct ncFifoHandle_t* fifoOut) { cv::Mat preprocessed_image_mat; preprocess_image(inputMat, preprocessed_image_mat); if (preprocessed_image_mat.rows != NETWORK_IMAGE_HEIGHT || preprocessed_image_mat.cols != NETWORK_IMAGE_WIDTH) { cout << "Error - preprocessed image is unexpected size!" << endl; return 0; } // three values for each pixel in the image. one value for each color channel RGB //float_t tensor32[3]; half_float tensor16[NETWORK_IMAGE_WIDTH * NETWORK_IMAGE_HEIGHT * 3]; float_t tensor32[NETWORK_IMAGE_WIDTH * NETWORK_IMAGE_HEIGHT * 3]; uint8_t* image_data_ptr = (uint8_t*)preprocessed_image_mat.data; int chan = preprocessed_image_mat.channels(); int tensor_index = 0; for (int col = 0; col < preprocessed_image_mat.cols; col++) { for (int row = 0; row < preprocessed_image_mat.rows; row++) { int pixel_start_index = col * (preprocessed_image_mat.rows + 0) * chan + row * chan; // TODO: don't hard code // assuming the image is in BGR format here uint8_t blue = image_data_ptr[pixel_start_index + 0]; uint8_t green = image_data_ptr[pixel_start_index + 1]; uint8_t red = image_data_ptr[pixel_start_index + 2]; // then assuming the network needs the data in BGR here. // also subtract the mean and multiply by the standard deviation from stat.txt file tensor32[tensor_index++] = (((float_t)blue - networkMean) * networkStd); tensor32[tensor_index++] = (((float_t)green - networkMean) * networkStd); tensor32[tensor_index++] = (((float_t)red - networkMean) * networkStd); /*tensor16[tensor_index++] = float2half(*((unsigned*)(&(tensor32[0])))); tensor16[tensor_index++] = float2half(*((unsigned*)(&(tensor32[1])))); tensor16[tensor_index++] = float2half(*((unsigned*)(&(tensor32[2]))));*/ } } // queue for inference unsigned int inputTensorLength = NETWORK_IMAGE_HEIGHT * NETWORK_IMAGE_WIDTH * 3 * sizeof(float_t); retCode = ncGraphQueueInferenceWithFifoElem(graphHandle, fifoIn, fifoOut, tensor32, &inputTensorLength, 0); if (retCode != NC_OK) { cout << "Error[" << retCode << "] - could not queue inference." << endl; return 0; } // get the size of the result unsigned int res_length; unsigned int option_length = sizeof(res_length); retCode = ncFifoGetOption(fifoOut, NC_RO_FIFO_ELEMENT_DATA_SIZE, &res_length, &option_length); if (retCode != NC_OK) { cout << "Error[" << retCode << "] - could not get output result size." << endl; return 0; } float_t result_buf[res_length]; retCode = ncFifoReadElem(fifoOut, result_buf, &res_length, NULL); if (retCode != NC_OK) { cout << "Error[" << retCode << "] - could not get output result." << endl; return 0; } list<networkResult> *objectInferenceResults = new list<networkResult>(); float number = *result_buf; for (int n = 0; n < number; ++n) { float* pointer = result_buf + (7 + n * 7); if (pointer[2] > CONFIDENCE_THRESHOLD) { networkResult r; r.class_ID = pointer[1]; r.confidence = pointer[2] * 100; r.x1 = pointer[3] * NETWORK_IMAGE_WIDTH; r.y1 = pointer[4] * NETWORK_IMAGE_HEIGHT; r.x2 = pointer[5] * NETWORK_IMAGE_WIDTH; r.y2 = pointer[6] * NETWORK_IMAGE_HEIGHT; objectInferenceResults->push_back(r); } } return std::shared_ptr<list<networkResult>>(objectInferenceResults); } int main(int argc, char** argv) { // Camera and image frames VideoCapture capture; Mat imgIn; // Key to escape from main loop and close program const int breakKey = 27; // esc == 27 int key; // Struct that will hold inference results std::shared_ptr<list<networkResult>>(Result); // Set up the camera capture.open(CAM_SOURCE); capture.set(CV_CAP_PROP_FRAME_WIDTH, WINDOW_WIDTH); capture.set(CV_CAP_PROP_FRAME_HEIGHT, WINDOW_HEIGHT); // Set up the display window namedWindow(WINDOW_NAME, WINDOW_NORMAL); resizeWindow(WINDOW_NAME, WINDOW_WIDTH, WINDOW_HEIGHT); setWindowProperty(WINDOW_NAME, CV_WND_PROP_ASPECTRATIO, CV_WINDOW_KEEPRATIO); moveWindow(WINDOW_NAME, 0, 0); Point winTextOrigin(0, 20); // Initialize the NCS device(s) and network graphs and FIFO queues initNCS(); unsigned int frame_count = 0; //Initialize Mobilenet SSD graph with IO FIFOs initSSD(); // main loop while (true) { // If the user presses the break key exit the loop key = waitKey(1); if ((key & 0xFF) == breakKey) { break; } // Get a frame from the camera capture >> imgIn; if (frame_count++ >= SKIP_AFTER) { capture >> imgIn; frame_count = 0; } // Flip the image horizontally //flip(imgIn, imgIn, 1); //Inference Result = getInferenceResults(imgIn, ssd_graph_handle, ssd_fifo_in, ssd_fifo_out); // Draw labels and rectangles on the image putText(imgIn, "Press ESC to exit", winTextOrigin, FONT, 2, GREEN, 2); for (list<networkResult>::iterator it = Result->begin(); it != Result->end(); it++) { // Draw a rectangle around the detected face rectangle(imgIn, Point(it->x1, it->y1), Point(it->x2, it->y2), RED, 1, 8, 0); // print the age and gender text to the window putText(imgIn, LABELS[it->class_ID], Point(it->x2, it->y2), FONT, FONT_SIZE, BLACK, 3); } // Show the image in the window imshow(WINDOW_NAME, imgIn); } // end main while loop //Close device and deallocate graph closeNCS(); return 0; }

[idata]

@Sujeendran Can you send me a link to your entire code?

[idata]

Hi @Tome_at_Intel ,

I finally figured out the issue. All that was required was to resize the input Mat to 300x300 dimension and apply the mean and std changes. No other preprocessing was required. It's working fine now. Thanks for your response!

[idata]

@Sujeendran Glad you figured it out!