Software Archive
Read-only legacy content

R200 capture to bitmap in C# (color, depth )

Nicolas_S_3
Beginner
497 Views

Greetings,

This is really at the moment an exploration time for me , since i'm quite new with this type of photogrammetry devices.

At the moment i'm just trying to explore the basic capacities of the camera : taking a sample from a stream, and exposing the color and the depth of this sample. at the end of this post is the '' code '' i 've written ( and a link too ) .

* saving a bitmap from the color part was really easy : https://drive.google.com/file/d/0B-hhckWlLCoucVctVEpzUGVCVnc/view?usp=sharing

* however , i'm not sure the depth rendering is correct .

I tried two ways :

  1. The test with the  running example in the SDK was okay , still flickering in some places, but okay . 
  2. if it is normal as a '' Frame '' not to fully cover a front volume. the SDK streaming example runs at 60fps , so the eye might perceive a '' patchwork '' of partial covered Volume  as a whole ?
  3. So is that capture correct ?
  1. but on each cases i got this weird result . Even if we have some  '' noise ''  ?? that's a bit too regular for noises ??
  2. with both the cases,  another post in the same forum said the result in the arrays are distance from the camera  in millimeters. So i did a grey bitmap with grey representing the metric scale. And a got that result 
  3.  I found out that for a max value in those arrays  i got something around 2.000.000 mm , so around 2 kilometers ?

Do i miss something ? in the code ? Or in theory ?

Below i hope the needed extract of my code to unsderstand :

(or just follow that link to download the file : https://drive.google.com/file/d/0B-hhckWlLCouenFYLUZMdHBDajg/view?usp=sharing )

 

 

using System;
using System.Drawing;
using System.Drawing.Imaging;
using System.Windows.Media.Imaging;

namespace shoot_I.streaming
{
    class _snapshot
    {
        #region VARS
        PXCMSenseManager senseManager = PXCMSenseManager.CreateInstance();
        PXCMSession session;

        #endregion


        #region CTOR
        public _snapshot()
        {
            senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 30);
            senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 628, 468, 60);

            senseManager.Init();
            session = senseManager.QuerySession();
        }

        #endregion

        #region functions

        #region snapshot color

        /// <summary>
        /// take a single color photo 
        /// </summary>
        /// <returns>returns a bitmap</returns>
        public Bitmap ColorSnapshot()
        {
            Bitmap bmp = new Bitmap(10, 10);
            for (;;)
            {
                if (senseManager.AcquireFrame(true).IsError()) break;
                else
                {
                    // capture
                    PXCMCapture.Sample sample = senseManager.QuerySample();
                    PXCMImage MyImg = sample.color;

                    //test
                    PXCMImage.ImageData MyImgData;

                    MyImg.AcquireAccess(PXCMImage.Access.ACCESS_READ, out MyImgData);
                    bmp = MyImgData.ToBitmap(0, MyImg.info.width, MyImg.info.height);



                    // release
                    senseManager.ReleaseFrame();
                    break;
                }
            }

            ////should be used if we want to shut down the cam .
            //senseManager.Dispose();

            return bmp;





        }

        #endregion

        #region snapshot depth

        /// <summary>
        /// take a single depth grey color image shot
        /// </summary>
        /// <returns></returns>
        public WriteableBitmap DepthSnapshot()
        {



            for (;;)
            {
                if (senseManager.AcquireFrame(true).IsError()) break;
                else
                {
                    // capture
                    PXCMCapture.Sample sample = senseManager.QuerySample();
                    PXCMImage MyImg = sample.depth;

                    //test
                    PXCMImage.ImageData MyImgData;

                    MyImg.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out MyImgData);
                    WriteableBitmap WBmp = MyImgData.ToWritableBitmap(0, MyImg.info.width, MyImg.info.height, 96, 96);

                    // release
                    senseManager.ReleaseFrame();
                    return WBmp;
                    //break;
                }
            }

            ////should be used if we want to shut down the cam .
            //senseManager.Dispose();

            return null;
        }

        /// <summary>
        /// take a single depth color short array shot
        /// </summary>
        /// <returns></returns>
        public short[] DepthshortArray()
        {
            short[] depthPixel = new short[1];

            for (;;)
            {
                if (senseManager.AcquireFrame(true).IsError()) break;
                else
                {
                    // capture
                    PXCMCapture.Sample sample = senseManager.QuerySample();
                    PXCMImage depth = sample.depth;

                    //test
                    PXCMImage.ImageData DepthData;

                    depth.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH_RAW, out DepthData);
                    int dwith = depth.info.width;
                    int dheight = depth.info.height;
                    depthPixel = DepthData.ToShortArray(0, dwith * dheight);

                    depth.Dispose();

                    // release
                    senseManager.ReleaseFrame();
                    break;
                }
            }

            //should be used if we want to shut down the cam .
            //senseManager.Dispose();
            return depthPixel;
        }

        /// <summary>
        /// take a single depth color int array shot
        /// </summary>
        /// <returns></returns>
        public int[] DepthintArray()
        {
            //short[] depthPixel = new short[1];

            for (;;)
            {
                if (senseManager.AcquireFrame(true).IsError()) break;
                else
                {
                    // capture
                    PXCMCapture.Sample sample = senseManager.QuerySample();
                    PXCMImage depth = sample.depth;

                    //test
                    PXCMImage.ImageData DepthData;

                    depth.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH_F32, out DepthData);
                    int dwith = depth.info.width;
                    int dheight = depth.info.height;
                    int[] depthPixel = DepthData.ToIntArray(0, dwith * dheight);

                    depth.Dispose();

                    // release
                    senseManager.ReleaseFrame();
                    return depthPixel;
                    //break;
                }
            }

            //should be used if we want to shut down the cam .
            //senseManager.Dispose();
            return null;
        }
        #endregion

        #endregion
    }
}

 

0 Kudos
4 Replies
Nikolay_I_
Beginner
497 Views

Hi Nicolas,

I'm by far not a specialist, but let me make an educated guess. First of all, when I look at your color picture, it becomes no wonder for me why the depth image is so space: your objects are mostly black, grey or white, poorly textured. I've been mostly working with a lower fps, and I think there is indeed a bit to little points there, but that isn't really surprising.

Now, as to the functions ToIntArray and ToShortArray you've tried to use. There's indeed a bit of theory: the camera sort of sees in planes. AFAIU, it also makes educated guesses as to where a given patch of points belongs. These patches are supposed to lie on one of the set of planes parallel to the camera plane and - according to my experience - having integer z coordinates in mm. Now, the first parameter of the ToShortArray function is described as "The data plane index", which is why I'd guess you get the data for just the 0'th plane rather then mapping all the recognized point to the bitmap. I may be awfully wrong in details, but it seems to me that both ToIntArray and ToShortArray are not the functions of choice.

0 Kudos
Johannes_V_
New Contributor I
497 Views

Hello Nicolas,

I too have a R200 and experienced this behaviour in the depth images. After debugging I came to the conclusion that this is an artifact.

I explain a bit more :

The depth data I took from the R200 camera is in 16 bit depth. This means values ranging from 0 - 65535. By taking a look at a 16 bit histogram (this is not easy to find. you need to construct it on your own if you want to see it), i noticed that the "real" values are ranging from ~300 - 8000, where 1000 units is 1 meter.

There were aso values that took exactly value 10000, these are the "sparks" (the brightest flickering spots) you can see in the depth image. Sometimes there were also kicking in values from higher values, but thats only single pixels at most.

When you show this data as a grey image, the show method usually normalises the values. This means the lowest measuerd value is shown as black and the highest measuerd value is shown as white. If your interesting values are in the range from 400- 8000 (you can limit your range by saying 2000-6000 which means 2 - 6 meter) then 400 is shown as black, and 8000 is shown as white. If there is a single pixel with the value 30000 in the picture, the showing method also normalises this. this means 400 is black and 30000 is white. your normal values of max 8000 look really dark now, you most likely dont even see them on the screen.

 

What I did was a preprocessing of the depth images. I iterated over every single pixel and made every value lower than 400 equal to 400. then i made every pixel higher than 8000 in the value 8000. this helps a lot. after some time i noticed the sparks are exactly value 10000 and i simply drop frames where there is a single pixel with exactly value 10000 and take the frame from the last iteration (copy needed).

 

If this helps your question in any way please tell me and I try to give more details of the preprocessing above.

 

 

One more thing : the sparks (artifacts more or less) are reflections from the laser projector. if you hit a mirror or other smooth surface directly (90 degree) you can see more sparks. the occurence of this could be influenced by the shutter time. afaik there is no software that lets you change this value on the r200 (on the s200 or sr300 there is a software for this issue)

0 Kudos
jb455
Valued Contributor II
497 Views

Hi,

I don't have time atm to go through your code in full, but one thing I noticed is that you're using PIXEL_FORMAT_DEPTH_RAW on line 135 which gives you the raw values from the camera, before they're converted to mm. So if you use PIXEL_FORMAT_DEPTH instead you should have that array filled with actual mm values (which may or may not be accurate, as mentioned by Nikolay).

Also in your ToIntArray method you're using PIXEL_FORMAT_DEPTH_F32, which should give you 32 bit float values, then casting to int. So, I dunno, maybe you would prefer to keep the extra accuracy of the float values? Though in old versions of the SDK (maybe it's been fixed in the most recent one) the float values were the same as the int just with ".0" added to the end so you may not get anything extra!

Hope this helps!

James

 

0 Kudos
Nicolas_S_3
Beginner
497 Views

thanks for all the answers , and sorry for the long silence,  i was on something else AND lost the access to my account...

regarding the poor depth quality snapshot ,and black image result :
@nikolay & Johannes & James : indeed , now that you all mentionned it ,  since i'm using the command PIXEL_FORMAT_DEPTH to get the image , it is the raw capture , with all the measures errors .both images in the next link is a snapshot of a wall : the raw one , and the high contrasted  version : on the high contrasted snapshot you see appearing a chunk of wall.

raw snapshot :

raw snapshot reworked (++ luminosity ++ contrast )


Mentionning the raw data collection . Well , i'm using the algorithm of intel to remove the false measures . It works well ,
 except for the reflexions .. PhotoUtils in PXCMEnhancedPhoto.

Since what you said regarding the short[] array :
@nikolay : not sure about the planes , i still don't really get how it works , for now i'm supposing the plane index is used when you store multiple informations in one PXCMIMage. Typically , you take a color + depth snapshot in same time, you store in your PXCMImage the color in plane 0 and the depth in plane 1 . If you only store the color snapshot OR the depth snapshot in the image, you'll deal only with the plane 0 .
@johannes : yes, i totally missed some work on the short[] array . If you take the grey depth image below, i just scale the min value and max value of the array to 0-255 ( grey color values ) . A bit more elaborated is the depth RGB which allows to use 0-765 values.
R=0,G=0,B=0 to R=255,G=0,B=0
R=255,G=0,B=0 to R=255,G=255,B=0
R=255,G=255,B=0 to R=255,G=255,B=255

a bit off topic , but what would be visually wiser to use in for colorimetry scaling ? ( i had in mind  carthography and representing height with colors ) .


@james B. : yes the int casting is kinda useless.. so i removed it .

 


 

 PXCMSenseManager _SenseManager;
        PXCMSession _Session;

        PXCMCapture.Sample _Sample;
        PXCMImage _Image;
        PXCMImage.ImageData _ImageData;

        PXCMPhoto _PhotoIn;
        PXCMPhoto _PhotoOut;
        PXCMEnhancedPhoto.PhotoUtils _PhotoUtils;
        

        IOToolbox _IOToolbox;
        BitmapToolBox _BitMapToolBox;

        Bitmap ProcessBitmap;

 public RealSenseToolBox()
        {
            //init
            _IOToolbox = new IOToolbox();
            _BitMapToolBox = new BitmapToolBox();

            _SenseManager = PXCMSenseManager.CreateInstance();
            _SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 30);
            _SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 628, 468, 60);
            _SenseManager.Init();

            _Session = _SenseManager.QuerySession();
        }

public void SnapShot()
        {
            for (;;)
            {
                if (_SenseManager.AcquireFrame(true).IsError()) break;
                else
                {
                    //cleaning image directory...
                    _IOToolbox.CreateDirectory(EnhancedPhotoModel.ImgPath);
                    _IOToolbox.CleanDirectory(EnhancedPhotoModel.ImgPath);

                    //preparing capture
                    _Sample = _SenseManager.QuerySample();
                    _PhotoUtils = PXCMEnhancedPhoto.PhotoUtils.CreateInstance(_Session);

                    _PhotoIn = _Session.CreatePhoto();
                    _PhotoIn.ImportFromPreviewSample(_Sample);


                    //capturing depth
                    _PhotoOut = _PhotoUtils.EnhanceDepth(_PhotoIn, PXCMEnhancedPhoto.PhotoUtils.DepthFillQuality.HIGH);
                    _Image = _PhotoOut.QueryDepth();
                    

                    _Image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out _ImageData);

                    short[] depthPixel = _ImageData.ToShortArray(0, _Image.info.width* _Image.info.height);

                    //save
                    SaveDifferentDepth(depthPixel);
                    _PhotoIn.SaveXDM(EnhancedPhotoModel.ImgPath +"/"+ EnhancedPhotoModel.ImgName+".jpg");
                    EnhancedPhotoModel.MaxDepth = depthPixel.Max();
                    EnhancedPhotoModel.MinDepth = depthPixel.Min();

                    // release
                    _Image.ReleaseAccess(_ImageData);
                    _SenseManager.ReleaseFrame();

                    break;
                }
            }

 

well some of the code might not tell you anything,  ( references to all the ToolBoxes links ) . But you have the core of the code...


test images here :
color :

depth grey :

depth RGB :

short[] array values(in mm) : min=651,Max=20871


Anyway , thanks for all the hints given . I suppose new step would be to remove reflect reflect artifacts by supressing too hight values as you suggested Johann.

0 Kudos
Reply