Fill in and detect contour rectangles in Java OpenCV

Question

I have an initial starting image through some processing that looks like this

What I want to do it is to fill in the contours so it looks somewhat like this

and find the best fit parallelograms of the two (or more) squares which would let me get each one of the four bounding lines like this

If anyone could point me to the right functions that would help, but I can't find anything helpful. I've tried many distorted rectangle correctors but couldn't get them to work.

Heres current source code

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package javacvtesting;
import java.awt.FlowLayout;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import javax.imageio.ImageIO;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.imgproc.Imgproc;
import org.opencv.core.MatOfByte;
import org.opencv.core.MatOfInt;
import org.opencv.core.MatOfPoint;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.highgui.Highgui;
/**
 *
 * @author Arhowk
 */
public class JavaCVTesting {

    /**
     * @param args the command line arguments
     *//*
  */



    public static BufferedImage convert(Mat m){
        Mat image_tmp = m;

        MatOfByte matOfByte = new MatOfByte();

        Highgui.imencode(".png", image_tmp, matOfByte); 

        byte[] byteArray = matOfByte.toArray();
        BufferedImage bufImage = null;

        try {

            InputStream in = new ByteArrayInputStream(byteArray);
            bufImage = ImageIO.read(in);

        } catch (Exception e) {
            e.printStackTrace();
        }finally{
            return bufImage;
        }
    }
    public static Mat convert(BufferedImage i){
        BufferedImage image = i;
        byte[] data = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
        Mat mat = new Mat(image.getHeight(),image.getWidth(), CvType.CV_8UC3);
        mat.put(0, 0, data);
        return mat;
    }
    public static void show(BufferedImage i){
        JFrame frame = new JFrame();
        frame.getContentPane().setLayout(new FlowLayout());
        frame.getContentPane().add(new JLabel(new ImageIcon(i)));
        frame.pack();
        frame.setVisible(true);
    }
    public static void main(String[] args) {
        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
        Mat src = Highgui.imread("D:\\0_image.png");
        Imgproc.cvtColor(src, src, Imgproc.COLOR_BGR2HSV);
        Mat dest = new Mat();
      // Mat dest = new Mat(src.width(), src.height(), src.type());
        Core.inRange(src, new Scalar(58,125,0), new Scalar(256,256,256), dest);

        Mat erode = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3,3));
        Mat dilate = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5,5));
        Imgproc.erode(dest, dest, erode);
        Imgproc.erode(dest, dest, erode);

        Imgproc.dilate(dest, dest, dilate);
        Imgproc.dilate(dest, dest, dilate);

        List<MatOfPoint> contours = new ArrayList<>();

        Imgproc.findContours(dest, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
        Imgproc.drawContours(dest, contours, -1, new Scalar(255,255,0));

        Panel p = new Panel();
        p.setImage(convert(dest));
        p.show();
    }
}

Solution

• To determine the outer contours you may use findContours with mode RETR_EXTERNAL:

  List<MatOfPoint> contours = new ArrayList<>();
  Mat dest = Mat.zeros(mat.size(), CvType.CV_8UC3);
  Scalar white = new Scalar(255, 255, 255));
  
  // Find contours
  Imgproc.findContours(image, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
  
  // Draw contours in dest Mat
  Imgproc.drawContours(dest, contours, -1, white);
  

  Saving the obtained Mat you will get:

• To fill in the obtained contours:

  for (MatOfPoint contour: contours) 
      Imgproc.fillPoly(dest, Arrays.asList(contour), white);
  

  Saving the new resulting Mat you will get this:

• And to find the best fit rectangle for each contour:

  Scalar green = new Scalar(81, 190, 0);
  for (MatOfPoint contour: contours) {
      RotatedRect rotatedRect = Imgproc.minAreaRect(new MatOfPoint2f(contour.toArray()));
      drawRotatedRect(dest, rotatedRect, green, 4);
  }
  
  public static void drawRotatedRect(Mat image, RotatedRect rotatedRect, Scalar color, int thickness) {
      Point[] vertices = new Point[4];
      rotatedRect.points(vertices);
      MatOfPoint points = new MatOfPoint(vertices);
      Imgproc.drawContours(image, Arrays.asList(points), -1, color, thickness);
  }
  

Finally you get this resulting image: