Fiducials are visual markers or reference points designed to be recognizable by computer vision systems. They consist of distinctive patterns, symbols, or shapes (like QR codes ) that can be reliably detected and tracked by a camera.
Fiducials are used in augmented reality and motion tracking to determine position, orientation, or scale of objects in physical space.
Generative audiovisual objects in augmented reality
BoofCV is a library that provides methods for computer vision. It must also be installed by the Library Manager in its version for Processing. For example, BoofCV has functions to detect and track markers in a video stream. By making the code react when certain markers are detected, the basic operation of Reactable can be understood in a simplified way.
Augmented Reality Audiovisual Synth
In this sketch, fiducials are detected and two different objects placed according to their position in AR. Their position and the rotation of their individual 3D matrix is calculated. As an example, their z-positions are sent out via OSC messages.
In SuperCollider, two sound objects accepting OSC messages from Processing could be defined. When Processing finds fiducials, this SuperCollider sketch adjusts the sounds according to the distance of the markers.
/**
* Augmented Reality Map Viewer
*
* This sketch uses NyARToolkit to detect markers and display a 3D world map
* on top of them in augmented reality. It tracks camera input and allows
* interaction with a 3D map object using mouse coordinates.
*/
// Import necessary libraries
import jp.nyatla.nyar4psg.*; // NyARToolkit library for Processing
import processing.video.*; // Video library for camera access
// Global variables
Capture cam; // Camera capture object
MultiMarker nya; // AR marker detection object
PVector pointer; // A Vector that holds coordinates of mousepointer in 3D-space
PImage map; // PImage for the world map image
PShape maptable, stand, mapboard; // PShapes for the virtual map stand components
/**
* Setup function - runs once at the beginning
*/
void setup() {
// Set up window size and enable 3D rendering
size(640, 480, P3D);
// Camera setup options - uncomment the one you need based on your system
print(Capture.list()); // Print available camera options to console
// Camera initialization options for different operating systems
//cam = new Capture(this, 640, 480); // Use internal cam
//cam = new Capture(this, width, height, "UGREEN Camera"); // Use external cam with name
cam = new Capture(this, "pipeline: avfvideosrc device-index=1"); // OSX(Mac), external Camera
//cam = new Capture(this, "pipeline: ksvideosrc device-index=1"); // Windows, external Camera
//cam = new Capture(this, 1920, 1080, "pipeline: ksvideosrc device-index=0! image/jpeg, width=1920, height=1080, framerate=30/1 ! jpegdec ! videoconvert"); // Linux option
// See https://wp.nyu.edu/shanghai-ima-interaction-lab/how-to-capture-camera-in-catalina/ for more camera options
// Initialize the AR marker detection system
nya = new MultiMarker(this, width, height, "data/camera_para.dat", NyAR4PsgConfig.CONFIG_PSG);
nya.addARMarker("data/patt.hiro", 80); // Add a marker pattern with size 80mm
// Load the world map image from the data folder
map = loadImage("worldmap_smaller.jpg");
// Start the camera capture
cam.start();
// Create a virtual 3D object group to hold the map components
createMapObject();
}
/**
* Helper method to create the 3D map object with stand and board
*/
void createMapObject() {
// Create a group to hold all map components
maptable = createShape(GROUP);
// Create the stand (base) of the map holder
stand = createShape(BOX, 300, 200, 100);
stand.setFill(color(255, 255, 255, 100)); // Semi-transparent white
maptable.addChild(stand);
// Create the map board that will display the actual map image
mapboard = createShape(BOX, 300, -200, 1);
mapboard.translate(0, 0, 55); // Position it above the stand
mapboard.setFill(color(255, 255, 255, 255)); // Solid white background
mapboard.setTexture(map); // Apply the map texture
maptable.addChild(mapboard);
}
/**
* Draw function - runs continuously in a loop
*/
void draw() {
// Check if camera is available
if (cam.available() != true) {
return; // If camera hasn't started yet, skip this frame
}
// Process camera input
cam.read(); // Read the latest frame from camera
nya.detect(cam); // Analyze camera image for AR markers
// Set black background and draw camera feed
background(0);
nya.drawBackground(cam); // Draw the camera image as background
// Check if marker is detected
if (!nya.isExist(0)) {
return; // If no marker found, skip the rest of this frame
}
// Convert mouse position to 3D coordinates relative to the marker
pointer = nya.screen2ObjectCoordSystem(0, mouseX, mouseY);
// Begin drawing in 3D space aligned with the detected marker
nya.beginTransform(0);
// Draw the map table with the world map
shape(maptable);
// Draw an interactive pointer at mouse position
stroke(100, 100, 0); // Set stroke color to dark yellow
translate(10, 10, 100); // Adjust position in 3D space
fill(200); // Light gray fill
ellipse((int)pointer.x, (int)pointer.y, 20, 20); // Draw circle at mouse position
// End the 3D transformation
nya.endTransform();
}
import processing.video.*;
import jp.nyatla.nyar4psg.*;
Capture cam;
MultiMarker nya;
int amount = 11;
Blossom[] blossoms = new Blossom[amount];
void setup() {
size(640,360,P3D);
colorMode(HSB);
//cam=new Capture(this,640,360); // Internal Cam
cam=new Capture(this,640,360, "Live! Cam Sync HD VF0770");
nya=new MultiMarker(this,width,height,"data/camera_para.dat",NyAR4PsgConfig.CONFIG_PSG);
for (int i=0; i < amount; i++){
nya.addNyIdMarker(i,80);
blossoms[i] = new Blossom();
blossoms[i].setPoints();
}
cam.start();
lights();
}
void draw()
{
if (cam.available() !=true) {
return;
}
cam.read();
nya.detect(cam);
nya.drawBackground(cam);
// loop through all possible markers and attach a generative blossom ------------------------
for (int i=0; i < amount; i++){
if((nya.isExist(i))){
nya.beginTransform(i);
stroke(0,200);
strokeWeight(3); // start drawing in 3D of the detected marker
line(0,0,0,0,0,blossoms[i].getStemHeight()); // draw the stem
translate(0, 0, blossoms[i].getStemHeight()); // translate the blossom by the length of its stem
blossoms[i].rotateObject(); // rotate blossom
rotateX(blossoms[i].getRotation());
rotateY(blossoms[i].getRotation());
blossoms[i].drawObject(); // draw blossom
nya.endTransform(); // end drawing in 3D of the marker
}
}
}
import jp.nyatla.nyar4psg.*;
import processing.video.*;
import oscP5.*;
import netP5.*;
// Core components for AR marker detection, camera capture, and OSC communication
Capture cam;
MultiMarker nya;
OscP5 oscP5;
NetAddress myRemoteLocation;
PVector origin;
float[] rotation;
float x,y,z;
int currentMarkerID;
void setup(){
size(640,480,P3D);
// Initialize basic components and reference points
origin = new PVector(0,0,0);
rotation = new float[3];
//cam=new Capture(this,640,480); // use internal cam
//cam = new Capture(this, width, height, "Live! Cam Sync HD VF0770"); // use external cam
cam = new Capture(this, width, height, "pipeline: avfvideosrc device-index=0, width=640, height=360"); //OSX(Mac), external Camera
// Setup OSC communication with SuperCollider
oscP5 = new OscP5(this,12000); // Port number 12000
myRemoteLocation = new NetAddress("127.0.0.1",57120); // localhost SuperCollider
nya=new MultiMarker(this,width,height,"data/camera_para.dat",NyAR4PsgConfig.CONFIG_PSG);
// Start camera and register AR markers to track
cam.start();
nya.addNyIdMarker(0,80);
nya.addNyIdMarker(1,80);
}
void draw(){
if (cam.available() !=true) return;
// Capture and process camera frame for AR detection
cam.read();
nya.detect(cam);
background(0);
nya.drawBackground(cam);
// Process each marker if detected in the scene
currentMarkerID = 1;
for (int i=0; i < 2; i++){
if((nya.isExist(i))){
nya.beginTransform(i);
// Get position and rotation of marker in world coordinates
PMatrix3D transMat = nya.getMatrix(i);
x = transMat.multX(origin.x,origin.y,origin.z);
y = transMat.multY(origin.x,origin.y,origin.z);
z = transMat.multZ(origin.x,origin.y,origin.z);
rotation = getRotation(transMat);
println("Object " + currentMarkerID + " at world coordinates x: " + x + ", y: " + y + ", z: " + z);
println("Rotation: (" + rotation[0] + ", " + rotation[1] + ", " + rotation[2] + ")");
// Render different 3D objects based on marker ID and send OSC data
switch (i){
case 0:
strokeWeight(10);
stroke(255);
fill(0,150);
box(100);
// send z coordinate to SuperCollider
OscMessage zeroMessage = new OscMessage("/zero");
zeroMessage.add(z);
oscP5.send(zeroMessage, myRemoteLocation);
break;
case 1:
strokeWeight(0.7);
fill(255, 0, 0, 120);
stroke(255,200);
sphere(100);
// send z coordinate to SuperCollider
OscMessage firstMessage = new OscMessage("/first");
firstMessage.add(z);
oscP5.send(firstMessage, myRemoteLocation);
break;
}
nya.endTransform();
}
currentMarkerID +=1;
}
}
// Utility function to extract Euler rotation angles from transformation matrix
float[] getRotation(PMatrix3D _m){
PMatrix3D m = _m;
float[] rotationXYZ = new float[3];
float sy = sqrt(m.m00 * m.m00 + m.m10 * m.m10);
float x, y, z;
if (sy > 1e-6) {
x = atan2(m.m21, m.m22);
y = atan2(-m.m20, sy);
z = atan2(m.m10, m.m00);
} else {
x = atan2(-m.m12, m.m11);
y = atan2(-m.m20, sy);
z = 0;
}
rotationXYZ[0] = x;
rotationXYZ[1] = y;
rotationXYZ[2] = z;
return rotationXYZ;
}
