Object detection with batch inference

[nnain1]

The code is prepared to do batch inference for object detection.

// -----------------------------------------------------------------------------------------------------

        // --------------------------- 3. Configure input & output ---------------------------------------------

        // --------------------------- Prepare input blobs -----------------------------------------------------
        slog::info << "Preparing input blobs" << slog::endl;

        /** Taking information about all topology inputs **/
        InputsDataMap inputInfo(network.getInputsInfo());
        if (inputInfo.size() != 1) throw std::logic_error("Sample supports topologies with 1 input only");

        auto inputInfoItem = *inputInfo.begin();

        /** Specifying the precision and layout of input data provided by the user.
         * This should be called before load of the network to the device **/
        inputInfoItem.second->setPrecision(Precision::U8);
        inputInfoItem.second->setLayout(Layout::NCHW);

        std::vector<std::shared_ptr<unsigned char>> imagesData = {};
        std::vector<std::string> validImageNames = {};
        for (const auto & i : imageNames) {
            FormatReader::ReaderPtr reader(i.c_str());
            if (reader.get() == nullptr) {
                slog::warn << "Image " + i + " cannot be read!" << slog::endl;
                continue;
            }
            /** Store image data **/
            std::shared_ptr<unsigned char> data(
                    reader->getData(inputInfoItem.second->getTensorDesc().getDims()[3],
                                    inputInfoItem.second->getTensorDesc().getDims()[2]));
            if (data != nullptr) {
                imagesData.push_back(data);
                validImageNames.push_back(i);
            }
        }
        if (imagesData.empty()) throw std::logic_error("Valid input images were not found!");
        /** Setting batch size using image count **/
        network.setBatchSize(imagesData.size());
        size_t batchSize = network.getBatchSize();
        slog::info << "Batch size is " << std::to_string(batchSize) << slog::endl;

        // -----------------------------------------------------------------------------------------------------

        // --------------------------- 4. Loading model to the device ------------------------------------------
        slog::info << "Loading model to the device" << slog::endl;
        ExecutableNetwork executable_network = ie.LoadNetwork(network, FLAGS_d);
        // -----------------------------------------------------------------------------------------------------

        // --------------------------- 5. Create infer request -------------------------------------------------
        slog::info << "Create infer request" << slog::endl;
        InferRequest inferRequest = executable_network.CreateInferRequest();
        // -----------------------------------------------------------------------------------------------------

        // --------------------------- 6. Prepare input --------------------------------------------------------
        for (auto & item : inputInfo) {
            Blob::Ptr inputBlob = inferRequest.GetBlob(item.first);
            SizeVector dims = inputBlob->getTensorDesc().getDims();
            /** Fill input tensor with images. First b channel, then g and r channels **/
            size_t num_channels = dims[1];
            size_t image_size = dims[3] * dims[2];

            auto data = inputBlob->buffer().as<PrecisionTrait<Precision::U8>::value_type *>();
            /** Iterate over all input images **/
            for (size_t image_id = 0; image_id < imagesData.size(); ++image_id) {
                /** Iterate over all pixel in image (b,g,r) **/
                for (size_t pid = 0; pid < image_size; pid++) {
                    /** Iterate over all channels **/
                    for (size_t ch = 0; ch < num_channels; ++ch) {
                        /**          [images stride + channels stride + pixel id ] all in bytes            **/
                        data[image_id * image_size * num_channels + ch * image_size + pid] = imagesData.at(image_id).get()[pid*num_channels + ch];
                    }
                }
            }
        }

        // -----------------------------------------------------------------------------------------------------

        // --------------------------- 7. Do inference ---------------------------------------------------------
        size_t numIterations = 10;
        size_t curIteration = 0;
        std::condition_variable condVar;

        inferRequest.SetCompletionCallback(
                [&] {
                    curIteration++;
                    slog::info << "Completed " << curIteration << " async request execution" << slog::endl;
                    if (curIteration < numIterations) {
                        /* here a user can read output containing inference results and put new input
                           to repeat async request again */
                        inferRequest.StartAsync();
                    } else {
                        /* continue sample execution after last Asynchronous inference request execution */
                        condVar.notify_one();
                    }
                });

        /* Start async request for the first time */
        slog::info << "Start inference (" << numIterations << " asynchronous executions)" << slog::endl;
        inferRequest.StartAsync();

        /* Wait all repetitions of the async request */
        std::mutex mutex;
        std::unique_lock<std::mutex> lock(mutex);
        condVar.wait(lock, [&]{ return curIteration == numIterations; });

        // -----------------------------------------------------------------------------------------------------

        // --------------------------- 8. Process output -------------------------------------------------------
        
        // -----------------------------------------------------------------------------------------------------
		slog::info << "Processing output blobs" << slog::endl;
		OutputsDataMap outputsInfo(network.getOutputsInfo());
		if (outputsInfo.size() != 1) throw std::logic_error("Sample supports topologies with 1 output only");
		Blob::Ptr outputBlob = inferRequest.GetBlob(outputsInfo.begin()->first);
		std::string outputName;
		DataPtr outputInfo;
		for (const auto& out : outputsInfo) {
			if (out.second->getCreatorLayer().lock()->type == "DetectionOutput") {
				outputName = out.first;
				outputInfo = out.second;
			}
		}

		if (outputInfo == nullptr) {
			throw std::logic_error("Can't find a DetectionOutput layer in the topology");
		}

		const SizeVector outputDims = outputInfo->getTensorDesc().getDims();

		const int maxProposalCount = outputDims[2];
		const int objectSize = outputDims[3];

		if (objectSize != 7) {
			throw std::logic_error("Output item should have 7 as a last dimension");
		}

		if (outputDims.size() != 4) {
			throw std::logic_error("Incorrect output dimensions for SSD model");
		}

		/** Set the precision of output data provided by the user, should be called before load of the network to the device **/
		outputInfo->setPrecision(Precision::FP32);
		/** Validating -nt value **/
		const size_t resultsCnt = outputBlob->size() / batchSize;
		if (FLAGS_nt > resultsCnt || FLAGS_nt < 1) {
			slog::warn << "-nt " << FLAGS_nt << " is not available for this network (-nt should be less than " \
				<< resultsCnt + 1 << " and more than 0)\n            will be used maximal value : " << resultsCnt << slog::endl;
			FLAGS_nt = resultsCnt;
		}
		const float* detection = static_cast<PrecisionTrait<Precision::FP32>::value_type*>(outputBlob->buffer());

At output processing, outputsInfo.begin()->first is just for the first image. How to get results for next images.

My object detection is based on Tensorflow's SSD model.

[Shubha_R_Intel]

Dear naing, nyan,

You could do it exactly as https://github.com/opencv/dldt/blob/2019/inference-engine/samples/object_detection_sample_ssd/main.cpp shows. Your batch_size may be 1 or 100 but see how images are organized according to batch_id.

for (size_t batch_id = 0; batch_id < batchSize; ++batch_id) {
            addRectangles(originalImagesData[batch_id].get(), imageHeights[batch_id], imageWidths[batch_id], boxes[batch_id], classes[batch_id],
                          BBOX_THICKNESS);
            const std::string image_path = "out_" + std::to_string(batch_id) + ".bmp";
            if (writeOutputBmp(image_path, originalImagesData[batch_id].get(), imageHeights[batch_id], imageWidths[batch_id])) {
                slog::info << "Image " + image_path + " created!" << slog::endl;
            } else {
                throw std::logic_error(std::string("Can't create a file: ") + image_path);
            }
        }

Hope it helps,

Thanks,

Shubha