Detecting faces in your webcam stream with OpenCV/java and displaying back the results

In this tutorial for java we detected the faces on a file picture and display a square around it.

In this series of tutorials we learnt:

1. Creating windows in java and drawing in them
2. Capture the camera stream with OpenCV and display using Java libraries (awt/swing).

In this tutorial for C++ we detected the faces and eyes on a input stream from a camera.

Now we want to do the same thing but in Java. Basically translate this C++/OpenCV tutorial into Java/OpenCV.

Got stucked in several places. Even placed a stackoverflow post. Here is the final code (I didn't do the eyes detection but that it is pretty straightforward...). This is actually the 2nd post on the code. This one is an optimized version where I corrected one error (see comments below), changed the way to get the mat (from OpenCV) into BufferedImage (from Java) using the comment below (thank you!), and defined more meaningful objects... The initial older version is further below, for reference... By the way, check also this post benchmarking different methods on how to go from Mat to BufferedImage.

 /*  
  * Captures the camera stream with OpenCV  
  * Search for the faces  
  * Display a circle around the faces using Java
  */  
 import java.awt.*;  
 import java.awt.image.BufferedImage;  
 import java.awt.image.DataBufferByte;  
 import javax.swing.*;  
 import org.opencv.core.Core;  
 import org.opencv.core.Mat;  
 import org.opencv.core.MatOfRect;  
 import org.opencv.core.Point;  
 import org.opencv.core.Rect;  
 import org.opencv.core.Scalar;  
 import org.opencv.core.Size;  
 import org.opencv.highgui.VideoCapture;  
 import org.opencv.imgproc.Imgproc;  
 import org.opencv.objdetect.CascadeClassifier;  
 class My_Panel extends JPanel{  
      private static final long serialVersionUID = 1L;  
      private BufferedImage image;  
      // Create a constructor method  
      public My_Panel(){  
           super();   
      }  
      /**  
       * Converts/writes a Mat into a BufferedImage.  
       *   
       * @param matrix Mat of type CV_8UC3 or CV_8UC1  
       * @return BufferedImage of type TYPE_3BYTE_BGR or TYPE_BYTE_GRAY  
       */  
      public boolean MatToBufferedImage(Mat matBGR){  
           long startTime = System.nanoTime();  
           int width = matBGR.width(), height = matBGR.height(), channels = matBGR.channels() ;  
           byte[] sourcePixels = new byte[width * height * channels];  
           matBGR.get(0, 0, sourcePixels);  
           // create new image and get reference to backing data  
           image = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);  
           final byte[] targetPixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();  
           System.arraycopy(sourcePixels, 0, targetPixels, 0, sourcePixels.length);  
           long endTime = System.nanoTime();  
           System.out.println(String.format("Elapsed time: %.2f ms", (float)(endTime - startTime)/1000000));  
           return true;  
      }  
      public void paintComponent(Graphics g){  
           super.paintComponent(g);   
           if (this.image==null) return;  
            g.drawImage(this.image,10,10,this.image.getWidth(),this.image.getHeight(), null);  
           //g.drawString("This is my custom Panel!",10,20);  
      }  
 }  
 class processor {  
      private CascadeClassifier face_cascade;  
      // Create a constructor method  
      public processor(){  
           face_cascade=new CascadeClassifier("C:/USERDATA/MEDICAL/CODE/java webcam facedetect/bin/haarcascade_frontalface_alt.xml");  
           if(face_cascade.empty())  
           {  
                System.out.println("--(!)Error loading A\n");  
                 return;  
           }  
           else  
           {  
                      System.out.println("Face classifier loooaaaaaded up");  
           }  
      }  
      public Mat detect(Mat inputframe){  
           Mat mRgba=new Mat();  
           Mat mGrey=new Mat();  
           MatOfRect faces = new MatOfRect();  
           inputframe.copyTo(mRgba);  
           inputframe.copyTo(mGrey);  
           Imgproc.cvtColor( mRgba, mGrey, Imgproc.COLOR_BGR2GRAY);  
           Imgproc.equalizeHist( mGrey, mGrey );  
           face_cascade.detectMultiScale(mGrey, faces);  
           System.out.println(String.format("Detected %s faces", faces.toArray().length));  
           for(Rect rect:faces.toArray())  
           {  
                Point center= new Point(rect.x + rect.width*0.5, rect.y + rect.height*0.5 );  
                Core.ellipse( mRgba, center, new Size( rect.width*0.5, rect.height*0.5), 0, 0, 360, new Scalar( 255, 0, 255 ), 4, 8, 0 );  
           }  
           return mRgba;  
      }  
 }  
 public class window {  
      public static void main(String arg[]){  
       // Load the native library.  
       System.loadLibrary("opencv_java245");       
       String window_name = "Capture - Face detection";  
       JFrame frame = new JFrame(window_name);  
    frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);  
    frame.setSize(400,400);  
    processor my_processor=new processor();  
    My_Panel my_panel = new My_Panel();  
    frame.setContentPane(my_panel);       
    frame.setVisible(true);        
       //-- 2. Read the video stream  
        Mat webcam_image=new Mat();  
        VideoCapture capture =new VideoCapture(0);   
    if( capture.isOpened())  
           {  
            while( true )  
            {  
                 capture.read(webcam_image);  
              if( !webcam_image.empty() )  
               {   
                    frame.setSize(webcam_image.width()+40,webcam_image.height()+60);  
                    //-- 3. Apply the classifier to the captured image  
                    webcam_image=my_processor.detect(webcam_image);  
                   //-- 4. Display the image  
                    my_panel.MatToBufferedImage(webcam_image); // We could look at the error...  
                    my_panel.repaint();   
               }  
               else  
               {   
                    System.out.println(" --(!) No captured frame -- Break!");   
                    break;   
               }  
              }  
             }  
             return;  
      }  
 }  

PS.: Check out this video on the Android 3D display I build based on this stuff and the details here.
PS1: Click here to see the index of these series of posts on OpenCV
PS2: This is the older version:

 /*  
  * Captures the camera stream with OpenCV  
  * Search for the faces  
  * Display a circle around the faces  
  */  
 import java.awt.*;  
 import java.awt.image.BufferedImage;  
 import javax.swing.*;  
 import org.opencv.core.Core;  
 import org.opencv.core.Mat;  
 import org.opencv.core.MatOfRect;  
 import org.opencv.core.Point;  
 import org.opencv.core.Rect;  
 import org.opencv.core.Scalar;  
 import org.opencv.core.Size;  
 import org.opencv.highgui.VideoCapture;  
 import org.opencv.imgproc.Imgproc;  
 import org.opencv.objdetect.CascadeClassifier;  
 class My_Panel extends JPanel{  
      private static final long serialVersionUID = 1L;  
      private BufferedImage image;  
      private CascadeClassifier face_cascade;  
      // Create a constructor method  
      public My_Panel(){  
           super();   
           //String face_cascade_name = "/haarcascade_frontalface_alt.xml";  
           //String face_cascade_name = "/lbpcascade_frontalface.xml";  
           //-- 1. Load the cascades  
           //String str;  
           //str = getClass().getResource(face_cascade_name).getPath();  
           //str = str.replace("/C:","C:");  
           //face_cascade_name=str;  
           //face_cascade=new CascadeClassifier(face_cascade_name);  
           face_cascade=new CascadeClassifier("C:/USERDATA/MEDICAL/CODE/java webcam facedetect/bin/haarcascade_frontalface_alt.xml");  
           //face_cascade=new CascadeClassifier("C:/eclipse/opencv/data/haarcascades/haarcascade_frontalface_alt.xml");  
           if(face_cascade.empty())  
           {  
                System.out.println("--(!)Error loading A\n");  
                 return;  
           }  
           else  
           {  
                      System.out.println("Face classifier loooaaaaaded up");  
           }  
      }  
      private BufferedImage getimage(){  
           return image;  
      }  
      public void setimage(BufferedImage newimage){  
           image=newimage;  
           return;  
      }  
      /**  
       * Converts/writes a Mat into a BufferedImage.  
       *   
       * @param matrix Mat of type CV_8UC3 or CV_8UC1  
       * @return BufferedImage of type TYPE_3BYTE_BGR or TYPE_BYTE_GRAY  
       */  
      public BufferedImage matToBufferedImage(Mat matrix) {  
        int cols = matrix.cols();  
        int rows = matrix.rows();  
        int elemSize = (int)matrix.elemSize();  
        byte[] data = new byte[cols * rows * elemSize];  
        int type;  
        matrix.get(0, 0, data);  
        switch (matrix.channels()) {  
          case 1:  
            type = BufferedImage.TYPE_BYTE_GRAY;  
            break;  
          case 3:   
            type = BufferedImage.TYPE_3BYTE_BGR;  
            // bgr to rgb  
            byte b;  
            for(int i=0; i<data.length; i=i+3) {  
              b = data[i];  
              data[i] = data[i+2];  
              data[i+2] = b;  
            }  
            break;  
          default:  
            return null;  
        }  
        BufferedImage image2 = new BufferedImage(cols, rows, type);  
        image2.getRaster().setDataElements(0, 0, cols, rows, data);  
        return image2;  
      }  
      public void paintComponent(Graphics g){  
            BufferedImage temp=getimage();  
            g.drawImage(temp,10,10,temp.getWidth(),temp.getHeight(), this);   
      }  
      public Mat detect(Mat inputframe){  
           Mat mRgba=new Mat();  
           Mat mGrey=new Mat();  
           MatOfRect faces = new MatOfRect();  
           inputframe.copyTo(mRgba);  
           inputframe.copyTo(mGrey);  
           Imgproc.cvtColor( mRgba, mGrey, Imgproc.COLOR_BGR2GRAY);  
           Imgproc.equalizeHist( mGrey, mGrey );  
           face_cascade.detectMultiScale(mGrey, faces);  
           //face_cascade.detectMultiScale(mGrey, faces, 1.1, 2, 0|Objdetect.CASCADE_SCALE_IMAGE, new Size(30, 30), new Size(200,200) );  
           //face_cascade.detectMultiScale(mGrey, faces, 1.1, 2, 2//CV_HAAR_SCALE_IMAGE,  
           //          ,new Size(30, 30), new Size(200,200) );  
           System.out.println(String.format("Detected %s faces", faces.toArray().length));  
           for(Rect rect:faces.toArray())  
           {  
                Point center= new Point(rect.x + rect.width*0.5, rect.y + rect.height*0.5 );  
                Core.ellipse( mRgba, center, new Size( rect.width*0.5, rect.height*0.5), 0, 0, 360, new Scalar( 255, 0, 255 ), 4, 8, 0 );  
           }  
           return mRgba;  
           }  
 }  
 public class window {  
      public static void main(String arg[]){  
       // Load the native library.  
       System.loadLibrary("opencv_java245");       
       String window_name = "Capture - Face detection";  
       JFrame frame = new JFrame(window_name);  
    frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);  
    frame.setSize(400,400);  
    My_Panel my_panel = new My_Panel();  
    frame.setContentPane(my_panel);       
    frame.setVisible(true);        
       //-- 2. Read the video stream  
    BufferedImage temp;  
        Mat webcam_image=new Mat();  
        VideoCapture capture =new VideoCapture(0);   
    if( capture.isOpened())  
           {  
            while( true )  
            {  
                 capture.read(webcam_image);  
              if( !webcam_image.empty() )  
               {   
                    frame.setSize(webcam_image.width()+40,webcam_image.height()+60);  
                    //-- 3. Apply the classifier to the captured image  
                    // At this point I was wondering where this should be done.  
                    // I put it within the panel class, but maybe one could actually  
                    // create a processor object...  
                    webcam_image=my_panel.detect(webcam_image);  
                   //-- 4. Display the image  
                    temp=my_panel.matToBufferedImage(webcam_image);  
                    my_panel.setimage(temp);  
                    my_panel.repaint();   
               }  
               else  
               {   
                    System.out.println(" --(!) No captured frame -- Break!");   
                    break;   
               }  
              }  
             }  
             return;  
      }  
 }