CNN in latest version

[Sunwoo_L_]

Hi,

Actually, the system that I am using has updated DAAL to 2017.0.0.098 version. In the previous version, my CNN code worked well, but doesn't work in the new version. I revised my code referencing the header files but still it causes a segmentation fault.

training::TopologyPtr configureSimpleNet()
{
   training::TopologyPtr topology(new training::Topology());
/* 1. first conv. */
   SharedPtr<layers::convolution2d::Batch<> > conv2d1(new layers::convolution2d::Batch<>);
   conv2d1->parameter.kernelSizes = layers::convolution2d::KernelSizes(5,5);
   conv2d1->parameter.nKernels = 20;

/* 2. first max-pool. */
   SharedPtr<layers::maximum_pooling2d::Batch<> > pool2d1(new layers::maximum_pooling2d::Batch<>(4));
   pool2d1->parameter = maximum_pooling2d::Parameter(2,3,2,2,2,2,0,0);

/* 3. second conv. */
   SharedPtr<layers::convolution2d::Batch<> > conv2d2(new layers::convolution2d::Batch<>);
   conv2d2->parameter.kernelSizes = layers::convolution2d::KernelSizes(5,5);
   conv2d2->parameter.nKernels = 40;

/* 4. second max-pool. */
   SharedPtr<layers::maximum_pooling2d::Batch<> > pool2d2(new layers::maximum_pooling2d::Batch<>(4));
   pool2d2->parameter = maximum_pooling2d::Parameter(2,3,2,2,2,2,0,0);
   //configuration.push_back(LayerDescriptor(3, pool2d2, NextLayers(4)));

/* 5. first fully-connected layer. */
    SharedPtr<layers::fullyconnected::Batch<> > fullyconnectedLayer1(new layers::fullyconnected::Batch<>(1000));

    fullyconnectedLayer1->parameter.weightsInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
                                                             new initializers::uniform::Batch<>(-0.001, 0.001));

    fullyconnectedLayer1->parameter.biasesInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
                                                            new initializers::uniform::Batch<>(0, 0.5));

/* 6. second fully-connected layer. */
    SharedPtr<layers::fullyconnected::Batch<> > fullyconnectedLayer2(new layers::fullyconnected::Batch<>(1000));

    fullyconnectedLayer2->parameter.weightsInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
                                                             new initializers::uniform::Batch<>(0.5, 1));

    fullyconnectedLayer2->parameter.biasesInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
                                                            new initializers::uniform::Batch<>(0.5, 1));


/* 7. third fully-connected layer. */
    SharedPtr<layers::fullyconnected::Batch<> > fullyconnectedLayer3(new layers::fullyconnected::Batch<>(10));

    fullyconnectedLayer2->parameter.weightsInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
                                                             new initializers::uniform::Batch<>(0.5, 1));

    fullyconnectedLayer2->parameter.biasesInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
                                                            new initializers::uniform::Batch<>(0.5, 1));


/* 8. last layer. */
    SharedPtr<layers::softmax::Batch<> > softmaxLayer(new layers::softmax::Batch<>());

    topology->add(conv2d1);
    topology->add(pool2d1);
    topology->add(conv2d2);
    topology->add(pool2d2);
    topology->add(fullyconnectedLayer1);
    topology->add(fullyconnectedLayer2);
    topology->add(fullyconnectedLayer3);
    topology->add(softmaxLayer);
   return topology;
}

int main(int argc, char *argv[])
{
    //checkArguments(argc, argv, 1, &datasetFileName);

        float* testData = new float[94*28*28];
        float* testData2 = new float[1];
        testData2[0] = 1;

        for (int i=0; i < 94; i++)
                for (int j = 0; j < 28; j++)
                    for (int k = 0; k < 28; k++)
                        testData[i*28*28 + j*28 + k] = float(j) * float(k) + i;

        size_t nDim = 4, dims[] = {94, 1, 28, 28};
        size_t dims2[] = {1};

        SharedPtr<Tensor> bogusData(new HomogenTensor<float>(nDim, dims, (float*)testData));
        SharedPtr<Tensor> bogusData2(new HomogenTensor<float>(1,dims2,(float*)testData2));

        double t1, t2;
        t1 = getUnixTime();
        training::Batch<> trainingNet;

        training::TopologyPtr topology = configureSimpleNet();
        trainingNet.initialize(bogusData->getDimensions(), *topology);

        trainingNet.input.set(training::data, bogusData);
        trainingNet.input.set(training::groundTruth, bogusData2);

        SharedPtr<optimization_solver::sgd::Batch<float> > sgdAlgorithm(new optimization_solver::sgd::Batch<float>());

    /* Set learning rate for the optimization solver used in the neural network */

        float learningRate = 0.001f;
        sgdAlgorithm->parameter.learningRateSequence = NumericTablePtr(new HomogenNumericTable<double>(1, 1, NumericTable::doAllocate, learningRate));

    /* Set the optimization solver for the neural network training */
        trainingNet.parameter.optimizationSolver = sgdAlgorithm;
        trainingNet.parameter.batchSize = 1;

        trainingNet.compute();
        t2 = getUnixTime();
        cout<<"Epoch: "<<t2-t1<<endl;

    return 0;
}

Since there's no kind explanation about how to use the new classes or functions, I am struggling with this simple CNN. Could you guys take a look at this simple code and see what's wrong with it? Basically, I am trying to train a CNN on sample 94 x 1 x 28 x 28 data. But it causes a segmentation fault in trainingNet.initialize().

Thank you.

[VictoriyaS_F_Intel · Accepted solution]

Hello Sunwoo,

We are going to extend our documentation with the descriptions of neural network Model and Topology classes in the future releases of Intel DAAL. We are also going to improve the error handling functionality in the neural networks components to provide more information about the failure to the user.

To fix issues with the code please make following changes:

1. Replace layers::softmax::Batch<> with layers::loss::softmax_cross::Batch<> in the topology. Because to train the network you have to specify the loss layer as the last layer of the neural network.
2. Create the connections between the layers in the topology. training::Topology::add method returns the unique identifier of a layer within the topology. Please use those identifiers to setup the connections between the layers with addNext method:

    const size_t conv1 = topology->add(conv2d1);
    const size_t pool1 = topology->add(pool2d1);
    const size_t conv2 = topology->add(conv2d2);
    const size_t pool2 = topology->add(pool2d2);
    const size_t fc1   = topology->add(fullyconnectedLayer1);
    const size_t fc2   = topology->add(fullyconnectedLayer2);
    const size_t fc3   = topology->add(fullyconnectedLayer3);
    const size_t sm    = topology->add(softmaxLayer);
    topology->get(conv1).addNext(pool1);
    topology->get(pool1).addNext(conv2);
    topology->get(conv2).addNext(pool2);
    topology->get(pool2).addNext(fc1);
    topology->get(fc1)  .addNext(fc2);
    topology->get(fc2)  .addNext(fc3);
    topology->get(fc3)  .addNext(sm);

From the code I understand that you want to train a network using 94 training samples of size 1 x 28 x 28.

If my understanding is correct then you have to make several changes in the main function also:

1. The size of testData2 array have to be changed from 1 to 94. Because you need 1 label per sample to run the training.
2. batchSize parameter of the neural network should be specified before the call to trainingNet.initialize because initialize function uses the information about the batch size.

I attach a file that contains all the corrections and runs successfully with DAAL 2017 Gold.

Best regards,

Victoriya

[Sunwoo_L_]

Hi, Victoriya,

I ran the attached file on Edison but it returns the below error message.

terminate called after throwing an instance of 'daal::services::interface1::Exception'

  what():  Size of the dimension in input tensor is incorrect
Details:
Argument name: data

Abort

I don't see any issue in the code. Do you have any idea what would be the incorrect dimension in the code?

[VictoriyaS_F_Intel]

Hello Sunwoo,

Can you please run library_version_info.cpp example and post the output here?

Thank you,

Victoirya

[Sunwoo_L_]

Here is the output.

Major version:          2017
Minor version:          0
Update version:         0
Product status:         Product
Build:                  20160722
Name:                   Intel(R) Data Analytics Acceleration Library
Processor optimization: Intel(R) Advanced Vector Extensions

Thank you.

[Sunwoo_L_]

When I remove all the other layers but the first conv layer and the last sm layer, it works well. It seems to have some issues with multiple hidden layers.

By the way, can I see the weight matrix and activations in each layer? For example, if would like to see the weight matrix, the inputs and outputs in the first layer, which functions can I use? Sorry for this kind of question, but I don't see an explanation about this in any documents.

Thank you. 

[VictoriyaS_F_Intel]

Thank you, Sunwoo. Please give us some time to run the code with exactly the same version of the library that you are using.

To retrieve weights, biases and activations of the layer please add following lines into neural_net_dense_batch.cpp example:

1. Retrieve the identifiers of the layers that form the neural network:

LayerIds ids;
training::TopologyPtr topology = configureNet(&ids);

2. Retrieve the input object of the forward layer from the model of the neural network:

forward::Input *fc1Input = trainingModel->getForwardLayer(ids.fc1)->getLayerInput();

Here we retrieve the input object of the first fully-connected layer using identifier ids.fc1.

3. Retrieve the tensors that contain the activations, weights and biases from the input object of the layer:

TensorPtr fc1Data    = fc1Input->get(forward::data);
TensorPtr fc1Weights = fc1Input->get(forward::weights);
TensorPtr fc1Biases  = fc1Input->get(forward::biases);

I attach neural_net_dense_batch.cpp example that is modified to retrieve activations, weights and biases.

Best regards,

Victoriya

[Sunwoo_L_]

Thank you, Victoriya.

I have one more question. In the reference document, I only see ReLU and softmax. Can I use sigmoid as the last layer? How can I use the sigmoid function?

[VictoriyaS_F_Intel]

I have reproduced the exception with DAAL 2017. The exception happens because the topology created in configureSimpleNet() returns the tensor of zero size as the result of the second pooling layer.

The output size of the convolution and pooling layers is defined by following formula:

outputSize = (inputSize + 2 * padding - kernelSize) / stride

Using this formula for the first four layers in the topology we get:

1. conv2d1: inputSize = 28, padding = 0, kernelSize = 5, stride = 2 (default value of stride for convolution), outputSize = 12.
2. pool2d1: inputSize = 12, padding = 0, kernelSize = 2, stride = 2, outputSize = 6.
3. conv2d2: inputSize = 6, padding = 0, kernelSize = 5, stride = 2, outputSize = 1.
4. pool2d2: inputSize = 1, padding = 0, kernelSize = 2, stride = 2, outputSize = 0.

That's why we see an exception.

To fix this error you can change the parameters of the topology. For example, set the strides size for the convolution layers to 1:

conv2d1->parameter.strides = layers::convolution2d::Strides(1, 1);
...
conv2d2->parameter.strides = layers::convolution2d::Strides(1, 1);

Regarding the sigmoid function. You can use sigmoid function in the hidden layers in your topology through the logistic layer. Please see layers::logistic::Batch for the details. But the latest version of the library does not provide logistic loss function, so you cannot use the logistic loss layer as the last layer in your topology.

Best regards,

Victoriya

[Sunwoo_L_]

It seems a little bit odd to me that the sigmoid function is not supported in the latest version. Anyway, thanks a lot for your help!

[Sunwoo_L_]

Hi, 

To verify the underlying math, I printed out the values between a conv. layer and a softmax layer and the values looked strange.

Input is a 2x2 matrix, (1,1,1,1) and conv filter size is 2x2. I expect to see one activation value.

Since I don't have a way to check the output of the conv layer, I printed out the softmax layer's input data instead.

While the 2x2 weight matrix is (-0.272, -0.055, -0.055, 0.491) and the biase value is -0.272, the input data of the last softmax layer is -0.158.

I tried several times, but don't know how this number is calculated. Here is the code below. Can you let me know how the value is calculated? I assume the activation function is sigmoid. What's the default activation function in convolutional layers?

#include "daal.h"
#include "service.h"

#include <sys/time.h>

using namespace std;
using namespace daal;
using namespace daal::algorithms;
using namespace daal::algorithms::neural_networks;
using namespace daal::algorithms::neural_networks::layers;
using namespace daal::services;

training::TopologyPtr configureSimpleNet()
{
   training::TopologyPtr topology(new training::Topology());
/* 1. first conv. */
   SharedPtr<layers::convolution2d::Batch<> > conv2d1(new layers::convolution2d::Batch<>);
   conv2d1->parameter.kernelSizes = layers::convolution2d::KernelSizes(2,2);
   conv2d1->parameter.strides = layers::convolution2d::Strides(1, 1);
   conv2d1->parameter.nKernels = 1;

/* 1. last layer. */
    SharedPtr<layers::loss::softmax_cross::Batch<> > softmaxLayer(new layers::loss::softmax_cross::Batch<>());

    const size_t conv1 = topology->add(conv2d1);
    const size_t sm    = topology->add(softmaxLayer);
    topology->get(conv1)  .addNext(sm);
    return topology;
}

int main(int argc, char *argv[])
{
        float* testData = new float[1*4*4];
        float* testData2 = new float[1];

        for (int i=0; i < 1; i++)
        {
            testData2 = 1;
                for (int j = 0; j < 2; j++)
                    for (int k = 0; k < 2; k++)
                        testData[i*2*2 + j*2 + k] = 1;
        }

        size_t nDim = 4, dims[] = {1, 1, 2, 2};
        size_t dims2[] = {1};

        SharedPtr<Tensor> bogusData(new HomogenTensor<float>(nDim, dims, (float*)testData));
        SharedPtr<Tensor> bogusData2(new HomogenTensor<float>(1,dims2,(float*)testData2));

        training::Batch<> trainingNet;

        trainingNet.parameter.batchSize = 1; // change to 94 for faster training
        training::TopologyPtr topology = configureSimpleNet();
        trainingNet.initialize(bogusData->getDimensions(), *topology);

        trainingNet.input.set(training::data, bogusData);
        trainingNet.input.set(training::groundTruth, bogusData2);

        SharedPtr<optimization_solver::sgd::Batch<float> > sgdAlgorithm(new optimization_solver::sgd::Batch<float>());

    /* Set learning rate for the optimization solver used in the neural network */

        float learningRate = 0.001f;
        sgdAlgorithm->parameter.learningRateSequence = NumericTablePtr(new HomogenNumericTable<double>(1, 1, NumericTable::doAllocate, learningRate));

    /* Set the optimization solver for the neural network training */
        trainingNet.parameter.optimizationSolver = sgdAlgorithm;

        trainingNet.compute();

        training::ModelPtr trainingModel = trainingNet.getResult()->get(training::model);

        forward::Input *fc1Input = trainingModel->getForwardLayer(0)->getLayerInput();
        TensorPtr fc1Data = fc1Input->get(forward::data);
        TensorPtr fc1Weights = fc1Input->get(forward::weights);
        TensorPtr fc1Biases = fc1Input->get(forward::biases);
        printTensor(fc1Data, "Activations :");
        printTensor(fc1Weights, "Weights :");
        printTensor(fc1Biases, "Biases :");

        forward::Input *fc2Input = trainingModel->getForwardLayer(1)->getLayerInput();
        TensorPtr fc2Data = fc2Input->get(forward::data);
        printTensor(fc2Data, "Activations :");

    return 0;
}
                        

[VictoriyaS_F_Intel]

Hello Sunwoo,

Let us set up the common terminology prior we continue the discussion.

A model of a neural network is an object that contains a set of layers that form the network. The connections between the layers in the model are defined by the network’s topology.

There are different types of layers available in Intel DAAL:

• Fully-connected layer
• Convolutional layer
• Activation layers
• …

Please refer to our User and Reference Guide for the full list of the available layers: https://software.intel.com/sites/products/documentation/doclib/daal/daal-user-and-reference-guides/daal_prog_guide/GUID-79741719-0DEA-40A1-8DB7-6042F62F1342.htm

Sigmoid activation layer is supported in the library that uses a different name for it, logistic layer. Please have a look at Logistic Layer for the details.

A training model consists of the forward layers and their counterparts, backward layers. Parameters of the model, weights and biases, are evaluated during the neural network training.

A prediction model consists of the forward layers only and uses the parameters of the model evaluated during its training for inference.

Intel DAAL expect the last layer of the training model to be the loss layer.

Loss layers are the layers that use input data and ground truth to compute the error obtained by the neural network on the training data. This error is used in backpropagation training to evaluate weights and biases that minimize the error on the training data set.

To resolve numeric stability issues the library combines computations in softmax and cross-entropy loss function into a separate layer, softmax with cross entropy. The present version of the library does not provide logistic layer combined with computation of the loss value.

The library allows you to get access to the results of the layers. For example, to get the result of the convolution layer please use following code:    

services::SharedPtr<forward::Result> conv1Result = trainingModel->getForwardLayer(0)->getLayerResult();
TensorPtr conv1Values = conv1Result->get(forward::value);

You will see that the result of the convolution layer is the same as the input of the softmax_cross layer.

Please refer to the description of 2d convolution layer for the formula that is used to compute the values of result: https://software.intel.com/sites/products/documentation/doclib/daal/daal-user-and-reference-guides/daal_prog_guide/GUID-C09D758F-F01C-4C53-AC9C-22CCCAE9ADF6.htm

The library currently does not combine computation of the convolution with any activation.

If you want an activation layer such as ReLU and logistic after the convolution, please add it explicitly in your topology

Please, let us know, if we answered your questions.

Also, let us know, if you need additional details or a help in tuning of your Intel DAAL based deep learning application

Best regards,

Victoriya

[Ying_H_Intel]

Hi Sunwoo,

Add one piece of note:

The discussion https://software.intel.com/en-us/forums/intel-data-analytics-acceleration-library/topic/706630 may help you to explain the result

Input is a 2x2 matrix, (1,1,1,1) and conv filter size is 2x2. I expect to see one activation value.

Since I don't have a way to check the output of the conv layer, I printed out the softmax layer's input data instead.

While the 2x2 weight matrix is (-0.272, -0.055, -0.055, 0.491) and the biase value is -0.272, the input data of the last softmax layer is -0.158.

when weightsAndBiasesInitialized (= false by default). the layer will call specified initializer to fill its weight and biases values automatically. for example,

Input is a 2x2 matrix, (1,1,1,1) and conv filter size is 2x2.  directly conv2d output  (no activation functions.)

wights =

-0.270666
-0.0541559
-0.0544038
0.491888

bias = -0.270666

Two-dimensional convolution layer result :
-0.158

Best Regards,

Ying

[satvik_k_]

i have copied the same topology and also have added the conv2d1->parameter.strides = layers::convolution2d::Strides(1, 1); for both convolution layers but i m still getting this exception

Unhandled exception at 0x75C1C54F in neural_net_dense_batch.exe: Microsoft C++ exception: daal::services::interface1::Exception at memory location 0x002AF238.

Am i missing any statement somewhere..

this is the complete code

#include "daal.h"
#include "service.h"
#include "neural_net_dense_batch.h"

using namespace std;
using namespace daal;
using namespace daal::algorithms;
using namespace daal::algorithms::neural_networks;
using namespace daal::services;

/* Input data set parameters */
string trainDatasetFile = "../data/batch/neural_network_train.csv";
string trainGroundTruthFile = "../data/batch/neural_network_train_ground_truth.csv";
string testDatasetFile = "../data/batch/neural_network_test.csv";
string testGroundTruthFile = "../data/batch/neural_network_test_ground_truth.csv";

prediction::ModelPtr predictionModel;
prediction::ResultPtr predictionResult;

void trainModel();
void testModel();
void printResults();
training::TopologyPtr configureSimpleNet()
{
    training::TopologyPtr topology(new training::Topology());
    /* 1. first conv. */
    SharedPtr<layers::convolution2d::Batch<> > conv2d1(new layers::convolution2d::Batch<>);
    conv2d1->parameter.kernelSizes = layers::convolution2d::KernelSizes(5, 5);
    conv2d1->parameter.nKernels = 20;
    conv2d1->parameter.strides = layers::convolution2d::Strides(1, 1);

    /* 2. first max-pool. */
    SharedPtr<layers::maximum_pooling2d::Batch<> > pool2d1(new layers::maximum_pooling2d::Batch<>(4));
    pool2d1->parameter = maximum_pooling2d::Parameter(2, 3, 2, 2, 2, 2, 0, 0);

    /* 3. second conv. */
    SharedPtr<layers::convolution2d::Batch<> > conv2d2(new layers::convolution2d::Batch<>);
    conv2d2->parameter.kernelSizes = layers::convolution2d::KernelSizes(5, 5);
    conv2d2->parameter.nKernels = 40;
    conv2d2->parameter.strides = layers::convolution2d::Strides(1, 1);

    /* 4. second max-pool. */
    SharedPtr<layers::maximum_pooling2d::Batch<> > pool2d2(new layers::maximum_pooling2d::Batch<>(4));
    pool2d2->parameter = maximum_pooling2d::Parameter(2, 3, 2, 2, 2, 2, 0, 0);
    //configuration.push_back(LayerDescriptor(3, pool2d2, NextLayers(4)));

    /* 5. first fully-connected layer. */
    SharedPtr<layers::fullyconnected::Batch<> > fullyconnectedLayer1(new layers::fullyconnected::Batch<>(1000));

    fullyconnectedLayer1->parameter.weightsInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
        new initializers::uniform::Batch<>(-0.001, 0.001));

    fullyconnectedLayer1->parameter.biasesInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
        new initializers::uniform::Batch<>(0, 0.5));

    /* 6. second fully-connected layer. */
    SharedPtr<layers::fullyconnected::Batch<> > fullyconnectedLayer2(new layers::fullyconnected::Batch<>(1000));

    fullyconnectedLayer2->parameter.weightsInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
        new initializers::uniform::Batch<>(0.5, 1));

    fullyconnectedLayer2->parameter.biasesInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
        new initializers::uniform::Batch<>(0.5, 1));

    /* 7. third fully-connected layer. */
    SharedPtr<layers::fullyconnected::Batch<> > fullyconnectedLayer3(new layers::fullyconnected::Batch<>(10));

    fullyconnectedLayer2->parameter.weightsInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
        new initializers::uniform::Batch<>(0.5, 1));

    fullyconnectedLayer2->parameter.biasesInitializer = services::SharedPtr<initializers::uniform::Batch<> >(
        new initializers::uniform::Batch<>(0.5, 1));

    /* 8. last layer. */
    SharedPtr<layers::loss::softmax_cross::Batch<> > softmaxLayer(new layers::loss::softmax_cross::Batch<>());

    const size_t conv1 = topology->add(conv2d1);
    const size_t pool1 = topology->add(pool2d1);
    const size_t conv2 = topology->add(conv2d2);
    const size_t pool2 = topology->add(pool2d2);
    const size_t fc1 = topology->add(fullyconnectedLayer1);
    const size_t fc2 = topology->add(fullyconnectedLayer2);
    const size_t fc3 = topology->add(fullyconnectedLayer3);
    const size_t sm = topology->add(softmaxLayer);
    topology->get(conv1).addNext(pool1);
    topology->get(pool1).addNext(conv2);
    topology->get(conv2).addNext(pool2);
    topology->get(pool2).addNext(fc1);
    topology->get(fc1).addNext(fc2);
    topology->get(fc2).addNext(fc3);
    topology->get(fc3).addNext(sm);
    return topology;
}
int main()
{
    trainModel();

    testModel();

    printResults();

    return 0;
}

void trainModel()
{
    /* Read training data set from a .csv file and create a tensor to store input data */
    TensorPtr trainingData = readTensorFromCSV(trainDatasetFile);
    TensorPtr trainingGroundTruth = readTensorFromCSV(trainGroundTruthFile);

    /* Create an algorithm to train neural network */
    training::Batch<> net;

    /* Set the batch size for the neural network training */
    net.parameter.batchSize = 10;

    /* Configure the neural network */
    LayerIds ids;
    training::TopologyPtr topology = configureSimpleNet();
    net.initialize(trainingData->getDimensions(), *topology);

    /* Pass a training data set and dependent values to the algorithm */
    net.input.set(training::data, trainingData);
    net.input.set(training::groundTruth, trainingGroundTruth);

    /* Create stochastic gradient descent (SGD) optimization solver algorithm */
    SharedPtr<optimization_solver::sgd::Batch<float> > sgdAlgorithm(new optimization_solver::sgd::Batch<float>());

    /* Set learning rate for the optimization solver used in the neural network */

    float learningRate = 0.001f;
    sgdAlgorithm->parameter.learningRateSequence = NumericTablePtr(new HomogenNumericTable<double>(1, 1, NumericTable::doAllocate, learningRate));

    /* Set the optimization solver for the neural network training */
    net.parameter.optimizationSolver = sgdAlgorithm;

    /* Run the neural network training */
    net.compute();

    /* Retrieve training and prediction models of the neural network */
    training::ModelPtr trainingModel = net.getResult()->get(training::model);

    /* Get activations, weights and biases of the 1st fully-connected layer */
    forward::Input *fc1Input = trainingModel->getForwardLayer(ids.fc1)->getLayerInput();
    TensorPtr fc1Data = fc1Input->get(forward::data);
    TensorPtr fc1Weights = fc1Input->get(forward::weights);
    TensorPtr fc1Biases = fc1Input->get(forward::biases);

    printTensor(fc1Data, "Activations :");
    printTensor(fc1Weights, "Weights :");
    printTensor(fc1Biases, "Biases :");
    predictionModel = trainingModel->getPredictionModel<float>();
}

void testModel()
{
    /* Read testing data set from a .csv file and create a tensor to store input data */
    TensorPtr predictionData = readTensorFromCSV(testDatasetFile);

    /* Create an algorithm to compute the neural network predictions */
    prediction::Batch<> net;

    net.parameter.batchSize = predictionData->getDimensionSize(0);

    /* Set input objects for the prediction neural network */
    net.input.set(prediction::model, predictionModel);
    net.input.set(prediction::data, predictionData);

    /* Run the neural network prediction */
    net.compute();

    /* Print results of the neural network prediction */
    predictionResult = net.getResult();
}

void printResults()
{
    /* Read testing ground truth from a .csv file and create a tensor to store the data */
    TensorPtr predictionGroundTruth = readTensorFromCSV(testGroundTruthFile);

    printTensors<int, float>(predictionGroundTruth, predictionResult->get(prediction::prediction),
        "Ground truth", "Neural network predictions: each class probability",
        "Neural network classification results (first 20 observations):", 20);
}