Tutorial - Using Human Body Tracking

This tutorial shows how to use your ZED 3D camera to detect and track human bodies using the ZED Body Tracking module.

Getting Started

  • First, download the latest version of the ZED SDK.
  • Download the Body Tracking sample code in C++, Python or C#.

Code Overview

Open the camera

In this tutorial, we will use the Body Tracking AI module of the ZED SDK. As in previous tutorials, we create, configure and open the camera.

// Create ZED objects
Camera zed;
InitParameters initParameters;
initParameters.camera_resolution = RESOLUTION::HD720;
initParameters.depth_mode = DEPTH_MODE::ULTRA;
initParameters.sdk_verbose = true;

// Open the camera
ERROR_CODE zed_error = zed.open(initParameters);
if (zed_error != ERROR_CODE::SUCCESS) {
	std::cout << "Error " << zed_error << ", exit program.\n";
	return 1; // Quit if an error occurred
}

# Create ZED objects
zed = sl.Camera()
init_params = sl.InitParameters()
init_params.camera_resolution = sl.RESOLUTION.HD720
init_params.depth_mode = sl.DEPTH_MODE.ULTRA
init_params.sdk_verbose = True

# Open the camera
err = zed.open(init_params)
if err != sl.ERROR_CODE.SUCCESS:
    # Quit if an error occurred
    exit()

// Create a ZED camera object
Camera zed = new Camera(0);
// Set configuration parameters
InitParameters init_params = new InitParameters();
init_params.resolution = RESOLUTION.HD720; 
init_params.depthMode = DEPTH_MODE.ULTRA;

// Open the camera
ERROR_CODE err = zed.Open(ref init_params);
if (err != ERROR_CODE.SUCCESS)
    Environment.Exit(-1)

Enable 3D Object detection

In order to perform Body Tracking, you need to enable the Object Detection module. Before enabling object detection, we specify the ObjectDetectionParameters of the module. Note that the object tracking needs the positional tracking to be able to track the objects in the world reference frame.

// Define the Objects detection module parameters
ObjectDetectionParameters detection_parameters;
// Different models can be chosen, optimizing the runtime or the accuracy
detection_parameters.detection_model = DETECTION_MODEL::HUMAN_BODY_FAST;
// Run detection for every Camera grab
detection_parameters.image_sync = true;
// Enable tracking to detect objects across time and space
detection_parameters.enable_tracking = true;
// Optimize the person joints position, requires more computations
detection_parameters.enable_body_fitting = true;

// If you want to have object tracking you need to enable positional tracking first
if (detection_parameters.enable_tracking)
    zed.enablePositionalTracking();

# Define the Object Detection module parameters
obj_param = sl.ObjectDetectionParameters()
# Different model can be chosen, optimizing the runtime or the accuracy
obj_param.detection_model = sl.DETECTION_MODEL.HUMAN_BODY_FAST
# run detection for every Camera grab
obj_param.image_sync = True
# Enable tracking to detect objects across time and space
obj_param.enable_tracking = True
# Optimize the person joints position, requires more computations
obj_param.enable_body_fitting = True

# If you want to have object tracking you need to enable positional tracking first
if obj_param.enable_tracking:
    positional_tracking_param = sl.PositionalTrackingParameters()
    zed.enable_positional_tracking(positional_tracking_param)


// Define the Objects detection module parameters
ObjectDetectionParameters object_detection_parameters = new ObjectDetectionParameters();
// Different models can be chosen, optimizing the runtime or the accuracy
object_detection_parameters.detectionModel = sl.DETECTION_MODEL.HUMAN_BODY_FAST;
// Run detection for every Camera grab
object_detection_parameters.imageSync = true;
// Enable tracking to detect objects across time and space
object_detection_parameters.enableObjectTracking = true;

// Object tracking requires the positional tracking module
// Enable positional tracking
PositionalTrackingParameters trackingParams = new PositionalTrackingParameters();
// If you want to have object tracking you need to enable positional tracking first
err = zedCamera.EnablePositionalTracking(ref trackingParams);

Then we can start the module, it will load the model. This operation can take a few seconds. The first time the module is used, the model will be optimized for the hardware and will take more time. This operation is done only once.

cout << "Object Detection: Loading Module..." << endl;
returned_state = zed.enableObjectDetection(detection_parameters);
if (returned_state != ERROR_CODE::SUCCESS) {
    cout << "Error " << returned_state << ", exit program.\n";
    zed.close();
    return EXIT_FAILURE;
}

print("Object Detection: Loading Module...")
err = zed.enable_object_detection(obj_param)
if err != sl.ERROR_CODE.SUCCESS:
    print(repr(err))
    zed.close()
    exit(1)

Console.WriteLine("Object Detection: Loading Module...");
err = zedCamera.EnableObjectDetection(ref object_detection_parameters);
if (err != ERROR_CODE.SUCCESS)
    Environment.Exit(-1);

Capture Data

The object confidence threshold can be adjusted at runtime to select only the revelant skeletons depending on the scene complexity. Since the parameters have been set to image_sync, for each grab call, the image will be fed into the AI module and will output the detections for each frames.

// Set runtime parameter confidence to 40
ObjectDetectionRuntimeParameters detection_parameters_rt;
detection_parameters_rt.detection_confidence_threshold = 40;

Objects objects;

// Grab new frames and detect objects
while (zed.grab() == ERROR_CODE::SUCCESS) {
	err = zed.retrieveObjects(objects, detection_parameters_runtime);
	if (objects.is_new) {
		// Count the number of objects detected
		cout << objects.object_list.size() << " Person(s) detected\n";
		if (!objects.object_list.empty()) {
			auto first_object = objects.object_list.front();
			// Display the 3D keypoint coordinates of the first detected person    
			cout << " Keypoints 3D \n";
			for (int i = 0; i < first_object.keypoint.size(); i++) {
				auto &kp = first_object.keypoint[i];
				cout << "    " << kp.x << ", " << kp.y << ", " << kp.z << "\n";
			}
		}
	}
}

# Set runtime parameter confidence to 40
obj_runtime_param = sl.ObjectDetectionRuntimeParameters()
obj_runtime_param.detection_confidence_threshold = 40

objects = sl.Objects()

# Grab new frames and detect objects
while zed.grab() == sl.ERROR_CODE.SUCCESS:
    err = zed.retrieve_objects(objects, obj_runtime_param)

    if objects.is_new:
        # Count the number of objects detected
        obj_array = objects.object_list
        print(str(len(obj_array)) + " Person(s) detected\n")

        if len(obj_array) > 0:
            first_object = objects.object_list[0]
            # Display the 3D keypoint coordinates of the first detected person
            print("\n Keypoint 3D ")
            keypoint = first_object.keypoint
            for it in keypoint:
                print("    " + str(it))


// Set runtime parameter confidence to 40
ObjectDetectionRuntimeParameters obj_runtime_parameters = new ObjectDetectionRuntimeParameters();
obj_runtime_parameters.detectionConfidenceThreshold = 40;

Objects objects = new Objects();

// Grab new frames and detect objects
while (zed.Grab(ref runtimeParameters) == ERROR_CODE.SUCCESS) {
	err = zed.RetrieveObjects(ref objects, ref detection_parameters_rt);

	if (objects.isNew) {
		// Count the number of objects detected
		Console.WriteLine(objects.numObject + " Person(s) detected");
		if(objects.numObject > 0)   
			ObjectData firstObject = objects.objectData[0];
			// Display the 3D keypoint coordinates of the first detected person
			Console.WriteLine(" Keypoints 3D ");
			for (int i = 0; i < firstObject.keypoints.Length; i++)
			{
				var kp = firstObject.keypoints[i];
				Console.WriteLine(" " + kp.X + ", " + kp.Y + ", " + kp.Z);
			}
	}
}

Disable modules and exit

Once the program is over the modules can be disabled and the camera closed. This step is optional since the zed.close() will take care of disabling all the modules. This function is also called automatically by the destructor if necessary.

// Disable object detection and close the camera
zed.disableObjectDetection();
zed.close();
return 0;

# Disable object detection and close the camera
zed.disable_object_detection()
zed.close()

// Disable object detection and close the camera
zed.DisableObjectDetection();
zed.Close();

And this is it!

Next Steps

To detect human bodies in the scene and display their 3D skeletons over the live image, check the 3D Object Detection advanced sample code.