topic Hi Sam, in Software Archive

R200 Calibration Data

SMali10 — Thu, 22 Oct 2015 21:16:51 GMT

I apologize for asking this question again, but it's been over a month and I'm afraid the previous thread became too bloated.

The functions "QueryStreamProjectionParameters" and "QueryStreamProjectionParametersEx" do not appear to be working with the R200 camera. They either cause the application to crash, or will return "PXCM_STATUS_FEATURE_UNSUPPORTED" and "PXCM_STATUS_DATA_UNAVAILABLE" respectively.

I have followed the documentation and tried in both Unity and C# examples. Nothing in the documentation states this is exclusively for the F200.

Has anyone gotten calibration data from the R200? Or is this feature currently broken/unsupported?

Thanks!

Could you provide the code

Xusheng_L_Intel — Thu, 22 Oct 2015 23:25:39 GMT

Could you provide the code how you used? Thanks!

Hi David,

SMali10 — Fri, 23 Oct 2015 15:53:37 GMT

Hi David,

Here is the code from the documentation I was following to initialize the calibration:

/* device is an instance of the PXCMCapture.Device interface */
PXCMProjection projection=device.CreateProjection();
 
/* Get a calibration instance */
PXCMCalibration calib=projection.QueryInstance<PXCCalibration>();
...
 
/* Dispose the interface */
projection.Dispose();

Here is the C# script I used in Unity. It runs fine unless the line "QueryStreamProjectionParameters" is uncommented. This causes the application to crash.

using UnityEngine;
using System.Collections;
using UnityEngine.UI;
using RSUnityToolkit;

public class irStream : MonoBehaviour {
	private PXCMSenseManager psm;

	private Texture2D irTexture;
	private ushort[] irData;
	private byte[] imgData;

	public RawImage displayImage;

	void Start () {
		psm = PXCMSenseManager.CreateInstance();
		if (psm == null) {
			Debug.LogError ("SenseManager Initialization Failed");
			return;
		}

		psm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_LEFT, 628, 468, 60f);
		// FUN FACT: in order to access the right stream of the R200 you must enable the left stream first
		//psm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_RIGHT, 628, 468, 60f);

		pxcmStatus sts = psm.Init();
		if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR) {
			Debug.LogError ("PXCMSenseManager.Init Failed");
			OnDisable (); // Clean-up
			return;
		}

		irData = new ushort[640*468];
		imgData = new byte[628*468*4];

		PXCMCapture.Device camera = psm.captureManager.device;

		PXCMProjection projection = camera.CreateProjection();
		PXCMCalibration calib = projection.QueryInstance<PXCMCalibration>();

		PXCMCalibration.StreamCalibration leftCalib = new PXCMCalibration.StreamCalibration();
		PXCMCalibration.StreamTransform leftTrans = new PXCMCalibration.StreamTransform();

		//calib.QueryStreamProjectionParameters (PXCMCapture.StreamType.STREAM_TYPE_LEFT, out leftCalib, out leftTrans);

		projection.Dispose();
	}

	void Update () {
		if (psm == null) return;
		if (psm.AcquireFrame(true) != pxcmStatus.PXCM_STATUS_NO_ERROR) return;

		PXCMCapture.Sample sample = psm.QuerySample();

		if (sample != null)
		{
			PXCMImage irImageSource = sample.left;

			if (irImageSource != null)
			{
				// Create the Texture 2D object if it doesn't exist already
				if (irTexture == null)
				{
					irTexture = new Texture2D(irImageSource.info.width,
					                              irImageSource.info.height,
					                               TextureFormat.RGBA32, false);
					displayImage.texture = irTexture;
				}

				// Get the raw 16bit data from the infrared image
				PXCMImage.ImageData irImageData;
				irImageSource.AcquireAccess(PXCMImage.Access.ACCESS_READ,
				                        PXCMImage.PixelFormat.PIXEL_FORMAT_Y16,
				                        out irImageData);
				irImageData.ToUShortArray(0, irData);

				// despite the ir camera being 628 pixels wide, the raw bytes are stored with a "pitch" of 640
				// in the case of 16bit data, the pitch is returned as 1280
				int pitch = irImageData.pitches[0]/2;
				int width = irImageSource.info.width;
				int height = irImageSource.info.height;
				
				for(int y = 0; y < height; y++){
					for(int x = 0; x < width; x++){
						int iA = (height-y-1) * pitch + x;
						int iB = (y * width + x) * 4;
						
						byte intensity = (byte)(irData[iA]/4);// raw pixels range between 0-1024

						imgData[iB] = intensity;
						imgData[iB+1] = intensity;
						imgData[iB+2] = intensity;
						imgData[iB+3] = 255; // Alpha
					}
				}
				
				irTexture.LoadRawTextureData(imgData);
				irTexture.Apply();
				
				irImageSource.ReleaseAccess(irImageData);
			}
		}
		psm.ReleaseFrame();
	}

	void OnDisable()
	{
		if (psm == null) return;
		psm.Dispose();
	}
}

Finally I tried modifying the C# RawStreams project in order to see if it was an issue with Unity. I changed the code in "RawStreams.cs" to the following.

SetStatus("Init Started");
if (sm.Init() >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/* Reset all properties */
sm.captureManager.device.ResetProperties(PXCMCapture.StreamType.STREAM_TYPE_ANY);

// Start Find Calibration Code ------ New Code
PXCMProjection projection = sm.captureManager.device.CreateProjection();
PXCMCalibration calib = projection.QueryInstance<PXCMCalibration>();

PXCMCalibration.StreamCalibration calibData;
PXCMCalibration.StreamTransform calibTrans;

SetStatus(calib.QueryStreamProjectionParameters(PXCMCapture.StreamType.STREAM_TYPE_COLOR, out calibData, out calibTrans).ToString());
                    
projection.Dispose();
// End Find Calibration Code ------ End New Code

/* Set mirror mode */
PXCMCapture.Device.MirrorMode mirror = Mirror ? PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL : PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
sm.captureManager.device.SetMirrorMode(mirror);

//SetStatus("Streaming");

For the functions "QueryStreamProjectionParameters" and "QueryStreamProjectionParametersEx" this will return "PXCM_STATUS_FEATURE_UNSUPPORTED" and "PXCM_STATUS_DATA_UNAVAILABLE" respectively.

Please see my initial question for more information: https://software.intel.com/en-us/forums/realsense/topic/594070

Thanks again!

Hey David, any word? I

SMali10 — Mon, 02 Nov 2015 18:07:25 GMT

Hey David, any word? I believe I have followed the documentation exactly.

Thanks!

Sam, the usage is not right.

Xusheng_L_Intel — Wed, 04 Nov 2015 15:06:17 GMT

Sam, the usage is not right. Just need change a little bit and it will work. Here is the c++ code. C# code should be almost same, Thanks!

PXCCalibration *calib = m_pProjection->QueryInstance<PXCCalibration>();

PXCCalibration::StreamCalibration calibData={};

PXCCalibration::StreamTransform calibTrans = {};

m_sts[0] = (STATUS)calib->QueryStreamProjectionParameters(PXCCapture::StreamType::STREAM_TYPE_COLOR, &calibData, &calibTrans);

m_sts[0] = (STATUS)calib->QueryStreamProjectionParametersEx(PXCCapture::StreamType::STREAM_TYPE_COLOR, PXCCapture::Device::StreamOption::STREAM_OPTION_ANY,&calibData, &calibTrans);

Hi David,

SMali10 — Wed, 04 Nov 2015 19:12:38 GMT

Hi David,

It looks like that code is the same as what is in the documentation. This is the C# code in the documentation:

/* device is an instance of the PXCMCapture.Device interface */
PXCMProjection projection=device.CreateProjection();
 
/* Get a calibration instance */
PXCMCalibration calib=projection.QueryInstance<PXCCalibration>();
...
 
/* Dispose the interface */
projection.Dispose();

and this is my implementation in Unity:

PXCMCapture.Device camera = psm.captureManager.device;

PXCMProjection projection = camera.CreateProjection();
PXCMCalibration calib = projection.QueryInstance<PXCMCalibration>();

PXCMCalibration.StreamCalibration leftCalib = new PXCMCalibration.StreamCalibration();
PXCMCalibration.StreamTransform leftTrans = new PXCMCalibration.StreamTransform();

calib.QueryStreamProjectionParameters (PXCMCapture.StreamType.STREAM_TYPE_LEFT, out leftCalib, out leftTrans);

projection.Dispose();

Calling the function "QueryStreamProjectionParameters" causes Unity to crash. There is no returned status or error message. The application freezes and needs to be shut down from the task manager. This has been tested on two different machines.

From what I can tell my C# code does all of the exact same things as what you have posted. Could you please tell me which part of the usage is wrong? Is there some other way to initialize the calibration and transform containers? Do the streams, device or projection need to be initialized in some other way first?

Thanks again, I really appreciate it!

Sam, sorry, your codes are

Xusheng_L_Intel — Wed, 04 Nov 2015 20:24:24 GMT

Sam, sorry, your codes are almost right except you need make sure to enable depth stream before you use those two APIs. You can try those codes in rawstreams.cs. I have verified and it works for me.

//New Code ---make sure to enable depth stream
sm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);

SetStatus("Init Started");
if (sm.Init() >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/* Reset all properties */
sm.captureManager.device.ResetProperties(PXCMCapture.StreamType.STREAM_TYPE_ANY);

// Start Find Calibration Code ------ New Code
PXCMProjection projection = sm.captureManager.device.CreateProjection();
PXCMCalibration calib = projection.QueryInstance<PXCMCalibration>();

PXCMCalibration.StreamCalibration calibData;
PXCMCalibration.StreamTransform calibTrans;

SetStatus(calib.QueryStreamProjectionParameters(PXCMCapture.StreamType.STREAM_TYPE_COLOR, out calibData, out calibTrans).ToString());
                    
projection.Dispose();
// End Find Calibration Code ------ End New Code

/* Set mirror mode */
PXCMCapture.Device.MirrorMode mirror = Mirror ? PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL : PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
sm.captureManager.device.SetMirrorMode(mirror);

//SetStatus("Streaming");

Thanks David,

SMali10 — Thu, 05 Nov 2015 05:40:31 GMT

Thanks David,

I am able to get the calibration data for the Color and Depth cameras in C#. Interestingly the calibrated values for the Depth camera return the same values as "QueryDepthFocalLength" and "QueryDepthPrincipalPoint".

However calibration data for the Left and Right IR cameras are still not accessible.

I am assuming the Left camera calibration data is identical to the Depth camera, but what about the Right camera? Do the Left and Right cameras have the same principle point? If not will there be a way to get calibration data for the Right IR camera?

Thanks again, I really appreciate your help!

Sam, what did you mean? why

Xusheng_L_Intel — Thu, 05 Nov 2015 16:53:41 GMT

Sam, what did you mean? why did you think principalPoint and focalLength should be different? How did you access the calibration data for Left and Right IR camera? They only output IR streams. Thanks!

Sam M. wrote:

Thanks David,

I am able to get the calibration data for the Color and Depth cameras in C#. Interestingly the calibrated values for the Depth camera return the same values as "QueryDepthFocalLength" and "QueryDepthPrincipalPoint".

However calibration data for the Left and Right IR cameras are still not accessible.

I am assuming the Left camera calibration data is identical to the Depth camera, but what about the Right camera? Do the Left and Right cameras have the same principle point? If not will there be a way to get calibration data for the Right IR camera?

Thanks again, I really appreciate your help!

Okay, I think I understand

SMali10 — Thu, 05 Nov 2015 20:00:36 GMT

Okay, I think I understand now. For the "device.Query..." functions the documentation says: "The property value is the model fixed value, and not the device instance calibrated value."

My assumption was that "model fixed" was a generic value for all R200 cameras, and "device instance" were the specific, calibrated values for that individual device. At least, that's how I understood it from the documentation.

Now my understanding is that "model fixed" are the values at full resolution, and "device instance" are the values at cropped resolutions from the SDK. That makes a lot more sense.

As for the IR cameras. It makes sense that if stereo rectification is applied to both IR streams, they would have the same focal length and principle point as the depth stream. I've been trying to access the calibration values from the IR camera streams, in order to confirm this.

So I guess after all of this, my real question is; are the focal lengths and principle points of the two IR camera streams the same as the depth camera stream?

Thanks again!

You are correct. The focal

Xusheng_L_Intel — Thu, 05 Nov 2015 23:11:05 GMT

You are correct. The focal length and principle point of both IR cameras and depth are same.

Sam M. wrote:

Okay, I think I understand now. For the "device.Query..." functions the documentation says: "The property value is the model fixed value, and not the device instance calibrated value."

My assumption was that "model fixed" was a generic value for all R200 cameras, and "device instance" were the specific, calibrated values for that individual device. At least, that's how I understood it from the documentation.

Now my understanding is that "model fixed" are the values at full resolution, and "device instance" are the values at cropped resolutions from the SDK. That makes a lot more sense.

As for the IR cameras. It makes sense that if stereo rectification is applied to both IR streams, they would have the same focal length and principle point as the depth stream. I've been trying to access the calibration values from the IR camera streams, in order to confirm this.

So I guess after all of this, my real question is; are the focal lengths and principle points of the two IR camera streams the same as the depth camera stream?

Thanks again!

Awesome, thanks Daniel, that

SMali10 — Thu, 05 Nov 2015 23:23:42 GMT

Awesome, thanks Daniel, that's a huge help!

Two last things, a request and a bug report.

I've guesstimated that the IR cameras are about 70mm apart. But it would be great if in a future release we could retrieve the calibrated distance between the two IR cameras, like how we can between the depth and color cameras.

Lastly the function "QueryStreamProjectionParameters" still causes Unity to crash. Here is a Unity Script with the updated code:

using UnityEngine;
using System.Collections;
using RSUnityToolkit;

public class ColorCalib : MonoBehaviour {
	private PXCMSenseManager sm;
	
	void Start () {
		sm = PXCMSenseManager.CreateInstance();
		if (sm == null) {
			Debug.LogError ("SenseManager Initialization Failed");
			return;
		}

		sm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 0, 0, 0);
		sm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
		
		pxcmStatus sts = sm.Init();
		if (sts >= pxcmStatus.PXCM_STATUS_NO_ERROR) {
			sm.captureManager.device.ResetProperties(PXCMCapture.StreamType.STREAM_TYPE_ANY);
			
			PXCMProjection projection = sm.captureManager.device.CreateProjection();
			PXCMCalibration calib = projection.QueryInstance<PXCMCalibration>();
			
			PXCMCalibration.StreamCalibration calibData;
			PXCMCalibration.StreamTransform calibTrans;
			
			calib.QueryStreamProjectionParameters (PXCMCapture.StreamType.STREAM_TYPE_COLOR, out calibData, out calibTrans);
			
			projection.Dispose();
		} else {
			Debug.LogError ("PXCMSenseManager.Init Failed");
			OnDisable (); // Clean-up
			return;
		}
	}
	
	void Update () {

	}
	
	void OnDisable()
	{
		if (sm == null) return;
		sm.Dispose();
	}
}

Thanks again!

Hi Sam,

Kevin_K_2 — Wed, 02 Nov 2016 19:53:20 GMT

Hi Sam,

Did you ever get the code to work in C# in Unity? It is almost exactly 1 year later, and I am running into the same crash in Unity upon calling QueryStreamProjectionParameters (). I am using a RealSense RS300, with the most recent RealSense SDK for Unity and updated driver.

Things I have tried are:

1. Making sure both the depth and color streams are enabled.

2. Waiting several hundred frames before calling the function -- I thought maybe it took some time for the calibration parameters to be ready. No luck, the code is happily displaying depth data for several hundred frames, and then BOOM it crashes when QueryStreamProjectionParameters () is called.

3. Making sure I called device.ResetProperties (PXCMCapture.StreamType.STREAM_TYPE_ANY), as David did in his code.