KiboRPC Image Processing

Improving SampleAPK

6th Kibo-RPC Tutorial Video: 05 Improving SampleAPK

In this session, they will explain how to log, Saving NavCam Images,　AR tag Recognition, Correcting Image Distortion, Pattern Matching, Checking the Simulation Resalts, Debug Images.

Image Recognition Programing

Kibo-RPC Tutorial Video: 07 Image Recognition Programing

AR Recognition

• Recognize if and where an item illustation is in our image
  • OpenCV | Aruco.detectMarkers

import java.util.ArrayList;
import java.util.List;
 
import org.opencv.core.Mat;
import org.opencv.aruco.Dictionary;
 
// Detect AR
Dictionary dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_5X5_250); // Load predefined ArUco dict of 250 unique 5x5 markers
List<Mat> corners = new ArrayList<>();
Mat markerIds = new Mat();
Aruco.detectMarkers(image, dictionary, corners, markerIds); // Detect markers and store the corners and IDs
 
// Convert image to RGB color space
Imgproc.cvtColor(image, image, Imgproc.COLOR_GRAY2RGB);
 
// Draw detected markers on image
if (!markerIds.empty()) {
    Scalar green = new Scalar(0, 255, 0);
    Scalar red = new Scalar(255, 0, 0);
    Aruco.drawDetectedMarkers(image, corners); //, markerIds, green);
 
    // Draw marker ID label
    if (corners.size() > 0) {
        Mat firstCorner = corners.get(0);
        double x = firstCorner.get(0, 0)[0];
        double y = firstCorner.get(0, 0)[1];
        org.opencv.core.Point labelPos = new org.opencv.core.Point(x, y - 30); // Adjust the offset as needed
        int markerId = (int) markerIds.get(0, 0)[0];
        Imgproc.putText(image, "id = " + markerId, labelPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, red, 2);
    }
 
    Log.i(TAG, "Markers detected: " + markerIds.dump());
} else {
    Log.i(TAG, "No markers detected.");
}

Correcting Image Distortion

Use information on the camera matrix and lens distortion parameters to straighten the captured image.

• Obtain camera intrinsics and distortion coefficients from the API

  • api.getNavCamIntrinsics()
  • api.getDockCamIntrinsics();

• Populate the cameraMatrix using the intrinsics retrieved.

  • Typically, intrinsics[0] and intrinsics[1] are focal lengths (fx and fy), intrinsics[2] and intrinsics[3] are principal points (cx and cy), and 1 is the unit value.

import org.opencv.calib3d.Calib3d;
 
// Correct image distortion
// Get camera matrix and populate with camera intrinsics
Mat cameraMatrix = new Mat(3, 3, CvType.CV_64F);
cameraMatrix.put(0, 0, api.getNavCamIntrinsics()[0]);
 
// Get lens distortion parameters
Mat cameraCoefficients = new Mat(1, 5, CvType.CV_64F);
cameraCoefficients.put(0, 0, api.getNavCamIntrinsics()[1]);
cameraCoefficients.convertTo(cameraCoefficients, CvType.CV_64F);
 
// Undistort image
Mat undistortImg = new Mat();
Calib3d.undistort(image, undistortImg, cameraMatrix, cameraCoefficients);
 
api.saveMatImage(undistortImg, "image_with_markers.png");

Pattern matching

• Under src/main, add a new folder (assets)

  • Drag pictures from lost_item_images folder to assets

• Import the template image as a grayscale (black and white) image in Mat format

import java.io.IOException;
import java.io.InputStream;
 
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
 
import org.opencv.imgproc.Imgproc;
 

• Asset manager

import android.content.res.AssetManager;
 
 

• perform pattern matching while changing the size and angle of template

import org.opencv.core.Size;
import org.opencv.core.Core;
 
// Resize image
private Mat resizeImg(Mat img, int width){
    int height = (int) (img.rows() * ((double) width / img.cols()));
    Mat resizedImg = new Mat();
    Imgproc.resize(img, resizedImg, new Size(width, height));
 
    return resizedImg;
}
 
// Rotate image
private Mat rotImage(Mat img, int angle) {
    org.opencv.core.Point center = new org.opencv.core.Point(img.cols() / 2.0, img.rows() / 2.0);
    Mat rotatedMat = Imgproc.getRotationMatrix2D(center, angle, 1.0);
    Mat rotatedImg = new Mat();
    Imgproc.warpAffine(img, rotatedImg, rotatedMat, img.size());
 
    return rotatedImg;
}

    // Pattern matching
    Mat[] templates = loadTemplateImages(imageFileNames);
    
    // Load template images
    private Mat[] loadTemplateImages(String[] imageFileNames){
        Mat[] templates = new Mat[imageFileNames.length];
        for (int i = 0; i < imageFileNames.length; i++) {
            try {
                // Open template image file in Bitmap from the filename and convert to Mat
                InputStream inputStream = assetManager.open(imageFileNames[i]);
                Bitmap bitmap = BitmapFactory.decodeStream(inputStream);
                Mat mat = new Mat();
                Utils.bitmapToMat(bitmap, mat);
 
                // Convert to grayscale
                Imgproc.cvtColor(mat, mat, Imgproc.COLOR_BayerBG2GRAY);
                templates[i] = mat; // Assign to array of templates
                inputStream.close();
 
            } catch (IOException e) {
                e.printStackTrace();;
            }
        }