Kinect Progress

After a bit of fuddling with the best method to interface with the Kinect for this project, I've found OpenCV to be the best C++ API for this task. OpenCV is an open source API used to display and manipulate image data. Its relatively light weight and not as obfuscated as the Windows API, which means I can complete whole tasks much quicker. Also I have chosen not to use OpenGL, as its pipeline adds additional complexity to how images are loaded from the video stream.

A link to the OpenCV site is here http://opencv.willowgarage.com/wiki/

My first implementations have been capturing and displaying color and depth frames from the device.  I have begun implementing OpenCV's face and eye classifiers to detect a face in the video stream, I have implemented the classifier, however it has not detected a face and I am unsure why.

A link to a tutorial on implementing OpenCV classifiers can be found here http://opencv.itseez.com/doc/tutorials/objdetect/cascade_classifier/cascade_classifier.html

The main functions are the CVKinect::Init() Which begins the main loop, and the getColor/Depth/Face() functions.  I am also using OpenCV2.3.

#include <stdio.h>
#include <iostream>

#include <cv.hpp>
#include <highgui\highgui.hpp>
#include <objdetect\objdetect.hpp>
#include <imgproc\imgproc.hpp>

#include <NuiApi.h>

#define RGB_WIDTH 640
#define RGB_HEIGHT 480

#define DEPTH_WIDTH 320
#define DEPTH_HEIGHT 240

#define RGB_CHANNELS 3
#define RGBA_CHANNELS 4

class CVKinect
{
public:

CVKinect( );
~CVKinect( );

int getColor( );
int getDepth( );
int getSkeleton( );
int getFace( );

RGBQUAD Nui_ShortToQuad_Depth( unsigned short s );

HRESULT init( CVKinect * );
void uninit( );

private:

cv::String face_cascade_name;// = "haarcascade_frontalface_alt.xml";
cv::String eyes_cascade_name;// = "haarcascade_eye_tree_eyeglasses.xml";
cv::CascadeClassifier face_cascade;
cv::CascadeClassifier eyes_cascade;
cv::RNG rng;


IplImage* rgb;
IplImage* depth;
IplImage* skeleton;

HANDLE rgbStream;
HANDLE rgbEvent;

HANDLE depthStream;
HANDLE depthEvent;

HANDLE skeletonStream;
HANDLE skeletonEvent;
};


using namespace std;
using namespace cv;

unsigned char imageBuffer[DEPTH_WIDTH*DEPTH_HEIGHT*RGB_CHANNELS];

CVKinect myCVKinect;

CVKinect::CVKinect( )
{
rgbEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
rgbStream = 0;

depthEvent = CreateEvent( NULL, TRUE, FALSE, NULL );
depthStream = 0;

skeletonEvent = 0;//CreateEvent( NULL, TRUE, FALSE, NULL );

rng = RNG(12345);
face_cascade_name = "C:\\Program Files\\OpenCV2.3\\data\\haarcascades\\haarcascade_frontalface_alt.xml";
eyes_cascade_name = "C:\\Program Files\\OpenCV2.3\\data\\haarcascades\\haarcascade_eye_tree_eyeglasses.xml";
}

CVKinect::~CVKinect( )
{

}

HRESULT CVKinect::init( CVKinect * myCVKinect )
{
rgb = cvCreateImageHeader(
cvSize( RGB_WIDTH, RGB_HEIGHT ),
IPL_DEPTH_8U,
RGBA_CHANNELS);

depth = cvCreateImageHeader(
cvSize( DEPTH_WIDTH, DEPTH_HEIGHT ),
IPL_DEPTH_8U,
RGB_CHANNELS);

HRESULT hr = NuiInitialize( NUI_INITIALIZE_FLAG_USES_COLOR | NUI_INITIALIZE_FLAG_USES_DEPTH );

if ( FAILED ( hr ) )
{
printf( "Error: The kinect sensor failed to initialize!\n" );
Sleep(1000);
return hr;
}

hr = NuiImageStreamOpen( NUI_IMAGE_TYPE_COLOR,
NUI_IMAGE_RESOLUTION_640x480,
0,
2,
myCVKinect->rgbEvent,
&myCVKinect->rgbStream);

if ( FAILED ( hr ) )
{
printf( "Error: The color stream failed to open!\n" );
return hr;
}

hr = NuiImageStreamOpen( NUI_IMAGE_TYPE_DEPTH,
NUI_IMAGE_RESOLUTION_320x240,
0,
2,
myCVKinect->depthEvent,
&myCVKinect->depthStream );

if ( FAILED ( hr ) )
{
printf( "Error: The depth stream failed to open!\n" );
return hr;
}

while (1)
{
WaitForSingleObject( rgbEvent, INFINITE );
myCVKinect->getFace( );

WaitForSingleObject( depthEvent, INFINITE );
myCVKinect->getDepth( );

int c = cvWaitKey(1);

if ( c == 'ESC' )
break;
}
return 1;

}

void CVKinect::uninit( )
{
cvReleaseImageHeader(&rgb);
cvReleaseImageHeader(&depth);
cvDestroyWindow("Video");
cvDestroyWindow("Depth");
NuiShutdown( );
}

int CVKinect::getColor( )
{
const NUI_IMAGE_FRAME * imageFrame = NULL;
HRESULT hr = NuiImageStreamGetNextFrame( rgbStream, 0, &imageFrame );

if ( FAILED ( hr ) )
{
return hr;
}

INuiFrameTexture * texture = imageFrame->pFrameTexture;
NUI_LOCKED_RECT lr;
texture->LockRect( 0, &lr, NULL, 0 );

if ( lr.Pitch != 0 )
{
BYTE * buffer = (BYTE *) lr.pBits;
cvSetData( rgb, buffer, lr.Pitch );
}

NuiImageStreamReleaseFrame( rgbStream, imageFrame );

cvShowImage("RGB Image", rgb);

return 1;
}

int CVKinect::getDepth( )
{

const NUI_IMAGE_FRAME * imageFrame = NULL;
HRESULT hr = NuiImageStreamGetNextFrame( depthStream, 0, &imageFrame );

if ( FAILED ( hr ) )
{
return hr;
}

INuiFrameTexture * texture = imageFrame->pFrameTexture;
NUI_LOCKED_RECT lr;
texture->LockRect( 0, &lr, NULL, 0 );

//13bits of depth data, 3 bits of player data
if ( lr.Pitch != 0 )
{
unsigned short * buffer = (unsigned short *) lr.pBits;

for ( unsigned int i = 0; i < DEPTH_WIDTH*DEPTH_HEIGHT; i ++ )
{

RGBQUAD color = Nui_ShortToQuad_Depth( buffer[i] );

imageBuffer[i*RGB_CHANNELS] = 255 - color.rgbRed;
imageBuffer[(i*RGB_CHANNELS)+1] = 255 - color.rgbGreen;
imageBuffer[(i*RGB_CHANNELS)+2] = 255 - color.rgbBlue;

}


cvSetData( depth, imageBuffer, DEPTH_WIDTH*RGB_CHANNELS );

}


NuiImageStreamReleaseFrame( depthStream, imageFrame );
cvShowImage( "Depth Image", depth );

return 1;
}

int CVKinect::getSkeleton( )
{
return 1;
}

int CVKinect::getFace( )
{

const NUI_IMAGE_FRAME * imageFrame = NULL;
HRESULT hr = NuiImageStreamGetNextFrame( rgbStream, 0, &imageFrame );

if ( FAILED ( hr ) )
{
return hr;
}

INuiFrameTexture * texture = imageFrame->pFrameTexture;
NUI_LOCKED_RECT lr;
texture->LockRect( 0, &lr, NULL, 0 );

if ( lr.Pitch != 0 )
{
BYTE * buffer = (BYTE *) lr.pBits;
cvSetData( rgb, buffer, lr.Pitch );
}

NuiImageStreamReleaseFrame( rgbStream, imageFrame );

//IplImage rgb
Mat frame = rgb;
std::vector<cv::Rect> faces;
Mat frame_gray;

cvtColor( frame, frame_gray, CV_RGB2GRAY );
equalizeHist( frame_gray, frame_gray );

//-- Detect faces
face_cascade.detectMultiScale(
frame_gray,
faces,
1.1,
2,
0|CV_HAAR_SCALE_IMAGE,
Size(20, 20)
);

for( int i = 0; i < faces.size(); i++ )
{
Point center( faces[i].x + faces[i].width*0.5, faces[i].y + faces[i].height*0.5 );
ellipse( frame, center, Size( faces[i].width*0.5, faces[i].height*0.5), 0, 0, 360, Scalar( 255, 0, 255 ), 4, 8, 0 );

Mat faceROI = frame_gray( faces[i] );
std::vector<cv::Rect> eyes;

//-- In each face, detect eyes
eyes_cascade.detectMultiScale( faceROI, eyes, 1.1, 2, 0 |CV_HAAR_SCALE_IMAGE, Size(30, 30) );

for( int j = 0; j < eyes.size(); j++ )
{
  Point center( faces[i].x + eyes[j].x + eyes[j].width*0.5, faces[i].y + eyes[j].y + eyes[j].height*0.5 );
  int radius = cvRound( (eyes[j].width + eyes[j].height)*0.25 );
  circle( frame, center, radius, Scalar( 255, 0, 0 ), 4, 8, 0 );
}
}

cvSetData(rgb, frame.data, frame.step);
cvShowImage("Face Image", rgb);

return 1;
}

RGBQUAD CVKinect::Nui_ShortToQuad_Depth( unsigned short s )
{
    unsigned short RealDepth = NuiDepthPixelToDepth(s);
    //USHORT Player    = NuiDepthPixelToPlayerIndex(s);

    // transform 13-bit depth information into an 8-bit intensity appropriate
    // for display (we disregard information in most significant bit)
    // BYTE intensity = (BYTE)~(s >> 4);
unsigned char intensity1;
unsigned char intensity2;
unsigned char intensity3;

intensity1 = (unsigned char)(s >> 8);
intensity2 = (unsigned char)(s >> 9);
intensity3 = (unsigned char)(s >> 10);

    // tint the intensity by dividing by per-player values

    RGBQUAD color;

    color.rgbRed   = intensity1 ;
    color.rgbGreen = intensity2 ;
    color.rgbBlue  = intensity3 ;

    return color;
}

int main(int argc, char* argv[])
{
myCVKinect.init( & myCVKinect );

myCVKinect.uninit( );

myCVKinect.~CVKinect( );


}