Projekt Leaper
Obsah
Členové projektu
Vratislav Kalenda, Studia nových médií, e-mail: v.kalenda@gmail.com
Marta Kolárová, Studia nových médií, e-mail: martavkolarova@gmail.com
Motivace vzniku projektu a jeho význam
Původním záměrem bylo pomocí leap motion naprogramovat světelné diody - ovládané pohyby. Následně jsme se rozhodli využít leap motion i pro vytváření a modulování zvuku, tj. vytvořit pohyby ovládaný virtuální hudební nástroj v ideálním případě napojený na knihovnu samplovaných zvuků. A ještě kombinovaný s barevnými světelnými diodami, které informují o hraných tónech. Cílem byla zároveň možnost spoluhry více hráčů a vytváření harmonií.
V první fázi byl zvuk generován přímo počítačem (bylo potřeba nastavit temperované ladění a rozmezí oktáv), v závěrečné fázi se ale podařilo systém propojit s knihovnou zvuků GarageBand pomocí protokolu MIDI – v současné době je k dispozici využití veškerých nástrojů GarageBand.
Vlastní zdrojový kód
Upozornění! Vzhledem k omezení syntaxe wikisofia kopírujte prosím zdrojový kód přímo z editace zdroje stránky (nebo využijte možnosti si jej stáhnout), nikoliv z podoby, do jaké je intepretována při zobrazení na stránce. Přislušný úsek je v souladu se syntaxí české wikipedie umístěn mezi značkou <tt> a </tt>.
var lifx = require('./lifx'); var util = require('util'); var packet = require('./packet'); var leap = require('leapjs'); var midi = require('midi');
var lx = lifx.init();
lx.on('bulbstate', function(b) { //console.log('Bulb state: ' + util.inspect(b)); });
lx.on('bulbonoff', function(b) { //console.log('Bulb on/off: ' + util.inspect(b)); });
Number.prototype.map = function ( in_min , in_max , out_min , out_max ) {
return ( this - in_min ) * ( out_max - out_min ) / ( in_max - in_min ) + out_min;
}
// Set up a new output.
var output = new midi.output();
// Count the available output ports. var count = output.getPortCount();
console.log("Available ports: " + count)
// Get the name of a specified output port. output.getPortName(0);
// Open the first available output port. output.openPort(0);
// Send a MIDI message. output.sendMessage([176,22,1]);
var damperPedal = true;
var toneMap = { 'c' : [ 60,72,84], //'cis' : [277.18,554.36], 'd' : [ 62,74], //'dis' : [ 311.13,622.26], 'e' : [ 64,76], 'f' : [ 65,77], //'fis' : [ 369.00,739.98], 'g' : [67,79], //'gis' : [ 415.00,830.60], 'a' : [69,81], //'ais' : [233.08,466.17], 'h' : [71,83] }
var dateDiff = function(start,end){
return (end.getTime() - start.getTime())/1000;
}
var findTone = function(freq){
var diff = 9999999999;
var tone = null;
for(var toneName in toneMap){
//console.log("looking in " + toneName)
for(var i = 0; i < toneMap[toneName].length; i++){
var newDiff = Math.abs(freq- toneMap[toneName][i]);
//console.log("diff " + newDiff)
if(newDiff < diff){
diff = newDiff;
tone = {name : toneName, freq : toneMap[toneName][i]}
//console.log("Found " + toneName + i)
}
}
}
if(tone != null){
//console.log("Found " + tone.name + " " + tone.freq + "hz")
}else{
//console.log("No tone for " + freq)
}
return tone;
}
var midiMessages = { "noteOn" : parseInt('10010000', 2), "noteOff" : parseInt('10000000',2), "mode" : parseInt('10110000',2), "damper" : parseInt('01000000',2), "pedalOff" : 127, "pedalOn" : 0 }
var findToneByMidi = function(step){ var diff = 9999999999; var tone = null; var stepIndex = 0; var i = 0; for(i = 0; i< 3; i++){
for(var toneName in toneMap){ //console.log("looking in " + toneName)
if(toneMap[toneName].length > i){
var newDiff = Math.abs(step-stepIndex); //console.log("aaa" + newDiff) //console.log("diff " + newDiff) if(newDiff < diff){ diff = newDiff; tone = {name : toneName, freq : toneMap[toneName][i]} //console.log("Found " + toneName + i) } stepIndex++; }
}
}
if(tone != null){ //console.log("Found " + tone.name + " " + tone.freq + "hz") }else{ //console.log("No tone for " + freq) }
return tone; }
var InterpolatedVal = function(){ this.data = []; this.windowSize = 5; this.add = function(val){ this.data.unshift(val); //console.log("Pusging value " + val + " win size " + this.windowSize) if(this.data.length > this.windowSize){ this.data.pop(val) } //console.log("CData lenght " + this.data.length) } this.getValue = function(){ var result = 0.0; for(var i = 0; i < this.data.length; i++){ result += this.data[i]; } //console.log("value " + result / this.data.length) return result / this.data.length }
};
var Hand = function(){ this.grip = new InterpolatedVal(); this.grip.windowSize = 30; this.rotation = new InterpolatedVal(); this.rotation.windowSize = 10; this.bulb = null;
this.note = new InterpolatedVal(); };
var leftHand = new Hand(); var rightHand = new Hand();
var baudio = require('baudio');
var leftTone = 440;
var rightTone = 440;
var curLeftTone = leftTone;
var curRightTone = rightTone;
var note = null;
var n = 0; // var b = baudio({rate: 704000},function (t) { // //console.log(t) // //console.log(leftTone) // var x = Math.sin(2* (rightTone+0.01) * Math.PI * t) ;//+ Math.sin(n)); // var next_x = Math.sin(2* (curLeftTone+0.02) * Math.PI * t); // var a = Math.sin(2* (leftTone+0.01) * Math.PI * t) ; // // if(Math.abs(x) < 0.001 && x < next_x) { // // curLeftTone = leftTone // // x = 0; // // } // // if(Math.abs(a) < 0.001) { // // curRightTone = rightTone // // a = 0; // // } // //n += Math.sin(t); // return (x * rightHand.grip.getValue()) + (a * leftHand.grip.getValue()); // }); // b.play();
// var baudio = require('baudio'), tune = require('tune');
// var blackholesun = tune(String( // 'A4 E4 A5 D5 A5 E4 A4 C4 E4 A5 D5 . . . ' + // 'G3 D4 G4 D5 G4 D4 E3 F#3 C#4 F#4 C#5 . . . ' + // 'E3 F3 C4 F4 A#5 F4 C4 F3 E3 D4 E4 B5 . . .' // ).split(' '));
// var b = baudio(function(t) { // // black hole sun + some fx // return blackholesun(t) + Math.sin(2 * Math.PI * t); // }); // b.play();
leftBulb = {};
leftBulb.isOn = false;
rightBulb = {};
rightBulb.isOn = false;
lx.on('bulb', function(b) {
console.log('New bulb found: ' + b.name + " : " + b.addr.toString("hex"));
addr = b.addr.toString("hex");
if(addr == "d073d5001d70"){
console.log("Assigning left bulb");
leftBulb.bulb = lx.bulbs[addr];
leftHand.bulb = leftBulb
}else if(addr == "d073d50139d0"){
console.log("Assigning right bulb");
rightHand.bulb = rightBulb
rightBulb.bulb = lx.bulbs[addr];
}
});
lx.on('gateway', function(g) { console.log('New gateway found: ' + g.ip); });
lx.on('packet', function(p) { // Show informational packets switch (p.packetTypeShortName) { case 'powerState': case 'wifiInfo': case 'wifiFirmwareState': case 'wifiState': case 'accessPoint': case 'bulbLabel': case 'tags': case 'tagLabels': //case 'lightStatus': case 'timeState': case 'resetSwitchState': case 'meshInfo': case 'meshFirmware': case 'versionState': case 'infoState': case 'mcuRailVoltage': console.log(p.packetTypeName + " - " + p.preamble.bulbAddress.toString('hex') + " - " + util.inspect(p.payload)); break; default: break; } });
console.log("Keys:"); console.log("Press 1 to turn the lights on"); console.log("Press 2 to turn the lights off"); console.log("Press 3 to turn the lights a dim red"); console.log("Press 4 to turn the lights a dim purple"); console.log("Press 5 to turn the lights a bright white"); console.log("Press 6 to cycle forwards through colours"); console.log("Press 7 to cycle backwards through colours"); console.log("Press 8 to show debug messages including network traffic"); console.log("Press 9 to hide debug messages including network traffic"); console.log("Press letters a-m to request various status fields");
var stdin = process.openStdin(); process.stdin.setRawMode(true); process.stdin.resume();
var cycledColour = 0;
var exit = false;
stdin.on('data', function (key) { //process.stdout.write('Got key ' + util.inspect(key) + '\n'); switch (key[0]) { case 0x20: console.log("hit") if(note != null){ sendMidi(null,note.freq,127); } case 0x31: // 1 console.log("Lights on"); // lx.lightsOn('Bedroom Lamp'); // Can specify one bulb by name var b = lx.bulbs['d073d5014163']; lx.lightsOn(b); // lx.lightsOn(); break;
case 0x32: // 2 console.log("Lights off"); var b = lx.bulbs['d073d5014163']; lx.lightsOff(b); // lx.lightsOff(); break;
case 0x33: // 3 console.log("Dim red"); // BB8 7D0 lx.lightsColour(0x0000, 0xffff, 1000, 0, 0); break;
case 0x34: // 4 console.log("Dim purple"); //lx.lightsColour(0x0000, 0xffff, 500, 0, 0); lx.lightsColour(0xcc15, 0xffff, 500, 0, 0); break;
case 0x35: // 5 console.log("Bright white"); lx.lightsColour(0x0000, 0x0000, 0x8000, 0x0af0, 0x0513); break;
case 0x36: // 6 cycledColour = (cycledColour+100) & 0xffff; console.log("Colour value is " + cycledColour); lx.lightsColour(cycledColour, 0xffff, 0x0200, 0x0000, 0x0000); break;
case 0x37: // 7 cycledColour = (cycledColour-100) & 0xffff; console.log("Colour value is " + cycledColour); lx.lightsColour(cycledColour, 0xffff, 0x0200, 0x0000, 0x0000); break;
case 0x38: // 8 console.log("Enabling debug"); lifx.setDebug(true); break;
case 0x39: // 9 console.log("Disabling debug"); lifx.setDebug(false); break;
case 0x61: // a console.log("Requesting voltage"); var message = packet.getMcuRailVoltage(); lx.sendToAll(message); break;
case 0x62: // b console.log("Requesting power state"); var message = packet.getPowerState(); lx.sendToAll(message); break;
case 0x63: // c console.log("hit") if(note != null){ sendMidi(midiMessages['noteOn'],note.freq,127); } break;
case 0x64: // d console.log("Requesting wifi firmware state"); var message = packet.getWifiFirmwareState(); lx.sendToAll(message); break;
case 0x65: // e console.log("Requesting wifi state"); var message = packet.getWifiState({interface:2}); lx.sendToAll(message); break;
case 0x66: // f console.log("Requesting bulb label"); var message = packet.getBulbLabel(); lx.sendToAll(message); break;
case 0x67: // g console.log("Requesting tags"); var message = packet.getTags(); lx.sendToAll(message); break;
case 0x68: // h console.log("Requesting tag label for tag 1"); var message = packet.getTagLabels({tags:new Buffer([1,0,0,0,0,0,0,0])}); lx.sendToAll(message); break;
case 0x69: // i console.log("Requesting time"); var message = packet.getTime(); lx.sendToAll(message); break;
case 0x6a: // j console.log("Requesting info"); var message = packet.getInfo(); lx.sendToAll(message); break;
case 0x6b: // k console.log("Requesting reset switch state"); var message = packet.getResetSwitchState(); lx.sendToAll(message); break;
case 0x6c: // l console.log("Requesting mesh info"); var message = packet.getMeshInfo(); lx.sendToAll(message); break;
case 0x6d: // m console.log("Requesting mesh firmware"); var message = packet.getMeshFirmware(); lx.sendToAll(message); break;
case 0x03: // ctrl-c console.log("Closing..."); lx.close(); // Close the port when done. output.closePort(); process.stdin.pause(); exit = true; process.exit(); break;
} });
var lastMidiMessage = new Date();
function sendMidi(c,b1,b2){ var diff = dateDiff(lastMidiMessage,new Date()); if(c == null){ c = midiMessages['noteOn'] } console.log("Diff is " + diff); if(diff > 0.10){ lastMidiMessage = new Date(); console.log("Sending c:"+c + " n:" + b1 + " v:" + b2) output.sendMessage([c,b1,b2]); if(c == midiMessages['noteOn']){ setTimeout(function(){ console.log("Sending OFF c:"+midiMessages['noteOff'] + " n:" + b1 + " v:" + b2) output.sendMessage([midiMessages['noteOff'],b1,b2]); }, 20); }
} }
function vectorToString(vector, digits) {
if (typeof digits === "undefined") { digits = 1; } return "(" + vector[0].toFixed(digits) + ", " + vector[1].toFixed(digits) + ", " + vector[2].toFixed(digits) + ")";
}
var lightsOn = false;
var lastNote = null;
var previousFrame = null;
gripInterpol = new InterpolatedVal
toneInterpol = new InterpolatedVal
leap.loop(function(frame){
if (frame.hands.length > 0) { for (var i = 0; i < frame.hands.length; i++) { var hand = frame.hands[i]; var handString = "";
handString += "
handString += "Hand ID: " + hand.id + "
"; handString += "Type: " + hand.type + " hand" + "
"; handString += "Direction: " + vectorToString(hand.direction, 2) + "
"; handString += "Palm position: " + vectorToString(hand.palmPosition) + " mm
"; handString += "Grab strength: " + hand.grabStrength + "
"; handString += "Pinch strength: " + hand.pinchStrength + "
"; handString += "Confidence: " + hand.confidence + "
"; handString += "Arm direction: " + vectorToString(hand.arm.direction()) + "
"; handString += "Arm center: " + vectorToString(hand.arm.center()) + "
"; handString += "Arm up vector: " + vectorToString(hand.arm.basis[1]) + "
"; //console.log(tone)
//console.log(handString); var handModel = null; if(hand.type == "right"){ handModel = rightHand; }else{ handModel = leftHand; console.log(handModel.grip.getValue()); }
handModel.grip.add(hand.grabStrength); handModel.note.add(hand.palmPosition[1]) var value = handModel.note.getValue(); value = Math.max(value,60); value = Math.min(value,500); value = value.map(60,500,0,14) note = findToneByMidi(value);
if(handModel.grip.getValue() > 0.8 && damperPedal == true){ console.log("gripping " + handModel.grip.getValue()) //sendMidi(midiMessages["mode"],parseInt('00000001',2),80) sendMidi(midiMessages["mode"],midiMessages["damper"],midiMessages["pedalOff"]) damperPedal = false } else if(damperPedal == false && handModel.grip.getValue() < 0.8){ damperPedal = true console.log("letting go") sendMidi(midiMessages["mode"],midiMessages["damper"],midiMessages["pedalOn"]) //sendMidi(midiMessages["mode"],parseInt('00000001',2),0) }
if(lastNote != note.freq){ lastNote = note.freq; console.log(note.freq) }
if(note != null && hand.type == "right"){ // Send a MIDI message. //sendMidi(midiMessages['noteOn'],note.freq,127);
//rightTone = note.freq; //console.log("assigning right tone " + rightTone) }else if(note != null){ //leftTone = note.freq; //sendMidi(midiMessages['noteOn'],note.freq,127);
//console.log("assigning left tone " + leftTone) }
if (previousFrame && previousFrame.valid) { var translation = hand.translation(previousFrame); //handString += "Translation: " + vectorToString(translation) + " mm
";
var rotationAxis = hand.rotationAxis(previousFrame, 2); var rotationAngle = hand.rotationAngle(previousFrame); handModel.rotation.add(rotationAxis[0]); //console.log('rotation ' + handModel.rotation.getValue()) //handString += "Rotation axis: " + vectorToString(rotationAxis) + "
"; //console.log(handString) }
// Store frame for motion functions previousFrame = frame;
// if(handModel.grip.getValue() >= 1 && handModel.bulb.isOn == false){ // lx.lightsOn(handModel.bulb.bulb); // handModel.bulb.isOn = true; // }else if(handModel.grip.getValue() <= 0.2 && handModel.bulb.isOn == true){ // lx.lightsOff(handModel.bulb.bulb); // handModel.bulb.isOn = false; // }
} }
});