A Kallio Movie with generative sound

Video created at the open video make session of OKFestival</a href=”http://okfestival.org/”> 2012 in Helsinki.

Credits

I used material from Kallio Archive. Footage by Emmi Vainio (7 videos clips), Cvijeta Miljak (2 video clips), Kalle Kuisma (1 video clip), Minna Tarkka (3 video clip).
All used video material (CC BY).

Software

Processing for video remix controlling SuperCollider via OSC

Code

s.boot;
Ndef(\buzzer).clear

(
Ndef(\buzzer, {
    var w = 106;
    var h = 1;
    var nums = w*h;
    var amps      = \amps.kr(0!nums);
    var freqScale = (\freqScale.kr(1!nums) - 0.5) * 2;
    var trig     = \trigs.tr;
    var vowels = (Vowel(\o, [\bass, \tenor])!10).flat;
    var freqs = vowels.collect(_.freqs).flat;
    var widths = vowels.collect(_.widths).flat;
    var forms = Formant.ar((({[50, 100, 150, 180, 250] * LFNoise1.kr(Rand(0.1, 0.2).range(1, 1.01))}!20)).flop.flat.scramble * freqScale.linlin(0, 1, 1, 0.125), freqs * freqScale.linlin(0, 1, 0.25, 2.5), widths) * 0.1;

    amps = LeakDC.kr(amps, 0.9995) * EnvGen.kr(Env.perc(0.5, 0.5), doneAction: 0, gate: PulseDivider.kr(trig, 10, {10.rand}!nums));

    Splay.ar(forms * amps * 2).tanh;
}).play
)

(
var oldAmps, amps, freqScale;
OSCdef(\hohi, {|msg, time, addr, recvPort|
    amps = (msg[4..].clump(3).collect(_.sum / 3));
    freqScale = msg[4..];
    Ndef(\buzzer).setn(\amps, amps.abs, \freqScale, freqScale, \trig, 1);
}, "/colors");
)

//import processing.opengl.*;
import processing.video.*;

// OSC stuff
import oscP5.*;
import netP5.*;

Movie movie;
PImage frame;
boolean capture = false; boolean drawing = false;
int sizeX, sizeY, numPixels;
int numRasterX, numRasterY, numTColors;

int[] tColorCounts;
OscP5 oscP5;
NetAddress netAddr;

void setup() {
  size(1280, 760, P2D);

  numRasterX = 106; 
  numRasterY = 1;

  strokeWeight(0);
  stroke(128);

  movie = new Movie(this, "kallioMuted.mp4");

  movie.loop();

  numTColors = numRasterX * numRasterY * 3;
  tColorCounts = new int[numTColors];

  // start oscP5, listening for incoming messages at port 12000
  oscP5 = new OscP5(this,12000);
  // where to send messages to:
  netAddr = new NetAddress("127.0.0.1",57120);
}

void draw() {
 if(movie.available()) {
         if (!drawing){
    capture = true;
    movie.read();
    movie.loadPixels(); // Make its pixels[] array available

    sizeX = movie.width; sizeY = movie.height;
    numPixels = sizeX * sizeY;

    for (int i = 0; i < numRasterX * numRasterY * 3; i++) {
      tColorCounts[i] = 0;
    }

    for  (int i = 0; i < sizeX; i++){
      int rasterPosX = (int) (((double) i / (double) sizeX) * (double)numRasterX);

      for (int j = 0; j < sizeY; j++){
        int rasterPosY = (int) (((double) j / (double) sizeY) * (double)numRasterY);
        int pixelColor = movie.pixels[j*sizeX + i];
        for (int k = 0; k < 3; k++){
          // interleaved bgr 3-tuple x by y
          // masks: b == 0, g == 8, r == 16
          // sum up color values (i.e. "b" "g" "r")
          // this only works for (numPixels * 255 < maxInt) 

          tColorCounts[
          ((rasterPosY * numRasterX + rasterPosX) * 3 + k)
          ] += (pixelColor >> (k*8) & 0xff);
        }

      }
    }
    capture = false;
  }
     }
//////////////// draw stuff and send OSC messages
  if (!capture) {
    drawing = true;
    int w = width/numRasterX;
    int h = height/numRasterY;
    int numROIPixels = (int) ((double) numPixels / (double) (numRasterX * numRasterY));
    for (int i = 0; i < numRasterX; i++){
      for (int j = 0; j < numRasterY; j++){
        int where = (j * numRasterX + i)*3;
        fill(
          (float) tColorCounts[where + 2] / (float)numROIPixels,
          (float) tColorCounts[where + 1] / (float)numROIPixels,
          (float) tColorCounts[where + 0] / (float)numROIPixels
        );
        rect(i * w, j * h, w, h);
      }
    }
    image(movie, (1280-640) / 2, (700-480) / 2);
    // has to be called from draw method to not cause failures
    OscMessage msg = new OscMessage("/colors");
    msg.add(numRasterX); // dimensions
    msg.add(numRasterY); // dimensions
    msg.add(3); // num colors

    for (int i = 0; i < numTColors; i++) {
      msg.add((float) tColorCounts[i] / (float)numROIPixels / 255.0);
    }
    oscP5.send(msg, netAddr);
    drawing = false;
  }
}