3D Environments

First Person Perspective

Processing

The sensor we used in session 3 to control a virtual object can also be used to control a virtual camera. By placing sensor/ESP32/battery on a headset, head movements can be tracked an linked to the virtual camera. Here is an example in Processing. A 3D environment is populated with simple spheres as placeholders for potential virtual performance partners that may sound. They are defined in the class 'Sonatars'.

The first person camera is also defined by its own class ('FPCamera'). Orientation in the 3D environment is derived from the head movements of the user. Key strokes allow for moving forwards or backwards in space. Of course, the interactions for movement could be made interactive in a more advanced way by adding more interfaces to the setup.

FirstPerson Sketch

// Install UDP library by Stephane Cousot

import peasy.*;
//import peasy.org.apache.commons.math.geometry.Vector3D;
//import peasy.org.apache.commons.math.geometry.Rotation;
import oscP5.*;
import netP5.*;
import hypermedia.net.*;

//PeasyCam cam;
FirstPersonCamera camera;

OscP5 oscP5;
NetAddress myRemoteLocation;

PVector cameraPos;
int population = 50;

Sonatar[] sonicActors = new Sonatar[population];

UDP udp;  // Create a UDP object
int port = 8888;  // Set the port number to listen for incoming messages from the ESP32


float rotX = 0;
float rotY = 0;
float rotZ = 0;

void setup() {
  //size(1200, 800, P3D);
  fullScreen(P3D);
  camera = new FirstPersonCamera(this, 0, 0, 0);
  //camera = new FirstPersonCamera(this, 0, -100, 300);
  camera.setRotation(15, 0, 0);
  
  // Initialize OSC addresses for SuperCollider
  oscP5 = new OscP5(this,12000);    // Port number 12000
  myRemoteLocation = new NetAddress("127.0.0.1",57120); // localhost
  
  // Initialize the UDP connection and start listening for ESP32
  udp = new UDP(this, port);
  udp.listen(true);
  
  for(int i = 0; i < sonicActors.length; i++ ){
    sonicActors[i] = new Sonatar(random(-PI,PI), random(-PI,PI), random(300,1000));
    //sonicActors[i] = new Sonatar(PI, random(-PI,PI), random(300,1000));
    OscMessage initialMessage = new OscMessage("/new");
    oscP5.send(initialMessage, myRemoteLocation);
  }
}

void draw() {
  background(0);
  ambientLight(100, 100, 126);
  directionalLight(100, 130, 160, -1, 0, 0);
  camera.setRotation(rotZ, rotX, rotY);         // Rotate the camera angle according to the sensor data
  camera.update();
  
  
  for(int i = 0; i < sonicActors.length; i++ ){
    sonicActors[i].paint();
    
    //float [] myPosFloat = cam.getPosition();
    //PVector myPos = new PVector(myPosFloat[0], myPosFloat[1], myPosFloat[2]);
    //float dist = sonicActors[i].getDistance(myPos);
    
    OscMessage volMessage = new OscMessage("/loudness");
    volMessage.add(i);
    //volMessage.add(dist);
    oscP5.send(volMessage, myRemoteLocation);
    //println(dist);
  }
  
}

// Receive function for UDP. Triggered when UDP data is arriving no matter which port
void receive(byte[] data, String ip, int port) {

  String message = new String(data);    // Convert the byte array to a string
  String senderIP = ip;                 // We can use the sender IP to see who was sending the data
  int senderPort = port;                // We can use the sender's port number to filter by sender 

  // Split the received String to get the XYZ-parameters
  String[] values = split(message, ",");
  String[] valX = split(values[0], ":");
  String[] valY = split(values[1], ":");
  String[] valZ = split(values[2], ":");
  
  // Get the values and convert them from degrees to radians 
  rotX = radians(float(valX[1]));
  rotY = radians(float(valY[1]));
  rotZ = radians(float(valZ[1]));
  
  println("Rotation x: "+rotX+" y: "+rotY+" z: "+rotZ);
  
}


void keyPressed() {
  float moveAmount = 10;
  if (key == 'i') camera.moveForward(moveAmount);  
  if (key == 'k') camera.moveForward(-moveAmount); 
}

Sonators

class Sonatar {
  float phi, theta, dist;
  float x, y, z;
  
  PVector cartesian;
  
  Sonatar(float _phi, float _theta, float _dist){
     phi = _phi;
     theta = _theta;
     dist = _dist;
     cartesian = makeCartesian();
  }
  
  public void paint(){
      pushMatrix();
        translate(cartesian.x, cartesian.y, cartesian.z);
        sphere(50);
      popMatrix();
  }
  
  private PVector makeCartesian(){
     x = dist*sin(phi)*cos(theta);
     y = dist*sin(phi)*sin(theta);
     z = dist*cos(phi);
     PVector cart = new PVector(x,y,z);
     return cart;
  }
  
  public PVector getCartesian(){
     return cartesian; 
  }
  
  public float getDistance(PVector _myPos){
    float d = PVector.dist(cartesian, _myPos);
    float dLimit = constrain(d, 50, 500);
    float l = map(dLimit, 50, 500, 1, 0);
    return l;
  }
  
}

FPCamera

class FirstPersonCamera {
  PApplet parent;
  float eyeX, eyeY, eyeZ;       // Camera position
  float pitch = 0;              // X-axis rotation (up/down)
  float yaw = 0;                // Y-axis rotation (left/right)
  float roll = 0;               // Z-axis rotation (tilt)
  
  FirstPersonCamera(PApplet parent, float x, float y, float z) {
    this.parent = parent;
    this.eyeX = x;
    this.eyeY = y;
    this.eyeZ = z;
  }
  
  void update() {
    // Calculate look-at point
    float lookX = eyeX + cos(yaw) * cos(pitch);
    float lookY = eyeY + sin(pitch);
    float lookZ = eyeZ + sin(yaw) * cos(pitch);
    
    // Apply camera transformation
    parent.camera(
      eyeX, eyeY, eyeZ,           // Camera position
      lookX, lookY, lookZ,         // Look-at point
      sin(roll), cos(roll), 0      // Up vector (affected by roll)
    );
  }
  
  void rotateX(float amount) {
    pitch += amount;
    pitch = constrain(pitch, -PI/2.1, PI/2.1);
  }
  
  void rotateY(float amount) {
    yaw += amount;
  }
  
  void rotateZ(float amount) {
    roll += amount;
  }
  
  void setRotation(float pitchDeg, float yawDeg, float rollDeg) {
    /*pitch = radians(pitchDeg);
    yaw = radians(yawDeg);
    roll = radians(rollDeg);*/
    pitch = pitchDeg;
    yaw = yawDeg;
    roll = rollDeg;
  }
  
  void moveForward(float amount) {
    // Calculate the forward direction vector based on pitch and yaw
    float dirX = cos(yaw) * cos(pitch);
    float dirY = sin(pitch);
    float dirZ = sin(yaw) * cos(pitch);
    
    // Normalize the direction vector
    float len = sqrt(dirX*dirX + dirY*dirY + dirZ*dirZ);
    if (len > 0) {
      dirX /= len;
      dirY /= len;
      dirZ /= len;
    }
    
    // Move the camera in the forward direction
    eyeX += dirX * amount;
    eyeY += dirY * amount;
    eyeZ += dirZ * amount;
  }
  
  void displayInfo() {
    parent.pushMatrix();
    parent.hint(DISABLE_DEPTH_TEST);
    parent.camera();
    parent.noLights();
    parent.fill(255);
    parent.textSize(16);
    parent.textAlign(LEFT);
    parent.text("Camera position: (" + nf(eyeX, 0, 1) + ", " + nf(eyeY, 0, 1) + ", " + nf(eyeZ, 0, 1) + ")", 20, 20);
    parent.text("Pitch: " + nf(degrees(pitch), 0, 1) + "°", 20, 40);
    parent.text("Yaw: " + nf(degrees(yaw), 0, 1) + "°", 20, 60);
    parent.text("Roll: " + nf(degrees(roll), 0, 1) + "°", 20, 80);
    parent.text("Controls: WASD = Look, QE = Roll, R = Reset, 1-4 = Preset views", 20, 100);
    parent.hint(ENABLE_DEPTH_TEST);
    parent.popMatrix();
  }
}

TouchDesigner