Community
cancel
Showing results for 
Search instead for 
Did you mean: 
220 Views

Problem with extraction of image from the blob

I created small application with OpenVINO which takes 3-channel image and apply to it neural net. NN was trained for segmentation and must find ellipses. But in output blob I see white ellipse and many noise pixels. Could anybody help me find an error?

In applied archive there are:

  1. TestOpVnn.cpp - main program (one file in MSVS project) - this file attached singly 
  2. Neural net in IR format (zf_unet_224)
  3. Basic neural net in Keras format (h5+json)
  4. Input image (input.png)
  5. Expected output image (output.png)
  6. Output image which gave us OpenVINO program (output_image.bmp)

Images:

Input:

Expected output:

Real output:

 

My application code:

 

#include "pch.h"
#include <iostream>
#include <string>
#include <vector>

// Without next pragmas there are many warnings, how correctly fix them?
#pragma warning(push)
#pragma warning(disable:4251)
#pragma warning(disable:4275)
#include <inference_engine.hpp>
#pragma warning(pop)

#include <opencv2/opencv.hpp>

template<class T>
std::ostream & operator << (std::ostream &out, const std::vector<T> &vec)
{
	out << "{ ";
	for (size_t i = 0; i + 1 < vec.size(); ++i)
	{
		out << vec << ", ";
	}
	if (!vec.empty())
	{
		out << vec.back();
	}
	out << " }";
	return out;
}

namespace io_info
{
	template<class T>
	void i_print_info(const T & i_info)
	{
		std::cout << "    io_info->getPrecision() = " << i_info->getPrecision() << std::endl;
		std::cout << "    io_info->getLayout   () = " << i_info->getLayout() << std::endl;
		std::cout << "    io_info->name        () = " << i_info->name() << std::endl;
		std::cout << "    io_info->dims           = " << i_info->getTensorDesc().getDims() << std::endl;
		std::cout << std::endl;
	}
}

namespace OV_helper
{
/**
* @brief Sets image data stored in cv::Mat object to a given Blob object.
* @param orig_image - given cv::Mat object with an image data.
* @param blob - Blob object which to be filled by an image data.
* @param batchIndex - batch index of an image inside of the blob.
*/
template <typename T>
void matU8ToBlob(const cv::Mat& orig_image, InferenceEngine::Blob::Ptr& blob, int batchIndex = 0) {
	InferenceEngine::SizeVector blobSize = blob->getTensorDesc().getDims();
	const size_t width = blobSize[3];
	const size_t height = blobSize[2];
	const size_t channels = blobSize[1];
	T* blob_data = blob->buffer().as<T*>();

	cv::Mat resized_image(orig_image);
	if (static_cast<int>(width) != orig_image.size().width ||
		static_cast<int>(height) != orig_image.size().height) {
		cv::resize(orig_image, resized_image, cv::Size(width, height));
	}

	int batchOffset = batchIndex * width * height * channels;

	for (size_t c = 0; c < channels; c++) {
		for (size_t h = 0; h < height; h++) {
			for (size_t w = 0; w < width; w++) {
				blob_data[batchOffset + c * width * height + h * width + w] =
					resized_image.at<cv::Vec3b>(h, w);
			}
		}
	}
}

// Extracts 1-channel image from the FP32 blob
cv::Mat blobFP32_to_img1ch(const InferenceEngine::Blob::Ptr &blob, int batchIndex = 0)
{
	InferenceEngine::SizeVector blobSize = blob->getTensorDesc().getDims();

	std::cout << "    blobSize = " << blobSize << std::endl;

	const size_t width    = blobSize[3];
	const size_t height   = blobSize[2];
	const size_t channels = blobSize[1];

	std::cout << "    width    = " << width    << std::endl;
	std::cout << "    height   = " << height   << std::endl;
	std::cout << "    channels = " << channels << std::endl;

	if (channels != 1)
	{
		throw std::runtime_error("OV_helper::blobFP32_to_img1ch() : Invalid numbers of channels.");
	}

	cv::Mat img(height, width, CV_8UC1);
	float* blob_data = blob->buffer().as<float*>();

	if (   static_cast<int>(width ) != img.size().width 
		|| static_cast<int>(height) != img.size().height)
	{
		throw std::runtime_error("OV_helper::blobFP32_to_img1ch() : Invalid img sizes.");
	}

	int batchOffset = batchIndex * width * height * channels;

	for (size_t c = 0; c < channels; c++) {
		for (size_t h = 0; h < height; h++) {
			for (size_t w = 0; w < width; w++) {
				img.at<uchar>(h, w) = 255 * blob_data[batchOffset + c * width * height + h * width + w];
			}
		}
	}

	return img;
}

}  // namespace OV_helper

int main()
{
	try
	{
		// Read model name
		std::string model_name;
		std::cin >> model_name;
		std::string model_xml = model_name + ".xml";
		std::string model_bin = model_name + ".bin";
		std::cout << std::endl << "      net   source = " << "[ " + model_xml + ", " + model_bin + " ]" << std::endl;
		// Read image name
		std::string image_name;
		std::cin >> image_name;
		cv::Mat1f frame = cv::imread(image_name, cv::IMREAD_GRAYSCALE);
		std::cout << "      image source = " << image_name << std::endl << std::endl;

		// 1 - Read neural net
// Without next 5 strings net didn't load. How load the net correctly?
#ifdef WIN32
		auto extension_ptr = InferenceEngine::make_so_pointer<::InferenceEngine::IExtension>(L"cpu_extension_avx2.dll");
#else
		auto extension_ptr = make_so_pointer<::InferenceEngine::IExtension>("libcpu_extension.so");
#endif
		InferenceEngine::Core core;
		InferenceEngine::CNNNetReader network_reader;
		network_reader.ReadNetwork(model_xml);
		network_reader.ReadWeights(model_bin);

		// 2 - Get net from reader class
		auto network = network_reader.getNetwork();

		// 3 - Get i/o info
		InferenceEngine::InputsDataMap  input_info  = network.getInputsInfo();
		InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo();
		if (input_info.size() != 1)
		{
			throw std::runtime_error("item.size() != 1");
		}
		if (output_info.size() != 1)
		{
			throw std::runtime_error("item.size() != 1");
		}

		std::cout << "    input_info.size() = " << input_info.size() << " output_info.size() = " << output_info.size() << std::endl << std::endl;
		std::cout << "#input/output names: " << std::endl;
		std::string  input_name = input_info .begin()->first;
		std::string output_name = output_info.begin()->first;
		std::cout << "    input_info.begin() = " << input_name << " output_info.begin() = " << output_name << std::endl << std::endl;

		// 5 - Load net
		core.AddExtension(extension_ptr,"CPU");
		auto executable_network = core.LoadNetwork(network, "CPU");

		// 6 - Create an inference request
		auto infer_request = executable_network.CreateInferRequest();

		// Print input parameters
		std::cout << "#Input  info: " << std::endl;
		io_info::i_print_info(input_info.at(input_name));
		// Print output parameters
		InferenceEngine::DataPtr o_info = output_info.at(output_name);
		std::cout << "#Output info: " << std::endl;
		std::cout << "    io_info->getPrecision() = " << o_info->getPrecision ()           << std::endl;
		std::cout << "    io_info->getLayout   () = " << o_info->getLayout    ()           << std::endl;
		std::cout << "    io_info->getName     () = " << o_info->getName      ()           << std::endl;
		std::cout << "    io_info->dims           = " << o_info->getTensorDesc().getDims() << std::endl;
		std::cout << std::endl;

		auto input = infer_request.GetBlob(input_name);
		// Normalizing of image to [-1/2 .. 1/2)
		frame = frame / 256.f - 0.5f;
		OV_helper::matU8ToBlob<float>(frame, input);

		infer_request.SetBlob(input_name, input);

		// 8 - Infer request
		infer_request.Infer();

		// 9 - Parse an answer
		auto output = infer_request.GetBlob(output_name);
		auto img = OV_helper::blobFP32_to_img1ch(output);
		cv::imwrite("output_image.bmp", img);
	}
	catch (std::exception &e)
	{
		std::cout << "Exception raised: " << e.what() << std::endl;
	}

	std::cout << "Ok return.\n" << std::endl;
	return 0;
}

 

0 Kudos
1 Reply
Shubha_R_Intel
Employee
220 Views

Dear Corvid, Zabeth,

I don't know the answer by spot-checking your code but I can tell you how to debug this problem. You can dump intermediate layers within your OpenVino app before and after inference - to figure out where the "noisy" or problematic layer is introduced.

You can do this via network.AddOutput("network_name").   Please see the following Intel Developer Zone post which I answered, to get more details.

https://software.intel.com/en-us/forums/computer-vision/topic/816896

Hope it helps,

Thanks,

Shubha

Reply