Hi all. what I'm trying to do is basically stream my desktop to the teensy so I can play music visualizers on my computer and have them show up on ws2811 LEDs. Right now I am basically grabbing a screenshot, cropping it, and changing the contrast before sending it to the LEDs. I have edited Paul's movie2serial to do this. It works, but there is a bit of latency between events on the screen and events on the lights. I know it is impossible to have zero latency, but I also know that I'm out of my league with the code I cobbled together and one of you might see something really obvious I could fix that would help speed things up considerably. So here is the code, if you have any suggestions they would be very much appreciated. Thanks!
Code:
import processing.video.*;
import processing.serial.*;
import java.awt.Rectangle;
import g4p_controls.*;
import java.awt.image.BufferedImage;
import java.awt.*;
//create slider to change contrast
GSlider sdr;
int numPorts=0; // the number of serial ports in use
int maxPorts=24; // maximum number of serial ports
Serial[] ledSerial = new Serial[maxPorts]; // each port's actual Serial port
Rectangle[] ledArea = new Rectangle[maxPorts]; // the area of the movie each port gets, in % (0-100)
boolean[] ledLayout = new boolean[maxPorts]; // layout of rows, true = even is left->right
PImage[] ledImage = new PImage[maxPorts]; // image sent to each port
int errorCount=0;
float framerate=0;
PImage screenShot;
PImage screenShot_en;
PImage screenShot_crop;
void setup() {
sdr = new GSlider(this, 0, 0, 200, 100, 15);
sdr.setLimits(1.0, 0.0, 255.0);
String[] list = Serial.list();
delay(20);
println("Serial Ports List:");
println(list);
serialConfigure("COM3"); // change these to your port names
if (errorCount > 0) exit();
size(600,80); // create the window
//pg = createGraphics(600, 80);
screenShot = new PImage(1920,1080);
screenShot_en = new PImage(600,80);
screenShot_crop = new PImage(600,80);
}
// image2data converts an image to OctoWS2811's raw data format.
// The number of vertical pixels in the image must be a multiple
// of 8. The data array must be the proper size for the image.
void image2data(PImage image, byte[] data, boolean layout) {
int offset = 3;
int x, y, xbegin, xend, xinc, mask;
int linesPerPin = image.height / 8;
int pixel[] = new int[8];
for (y = 0; y < linesPerPin; y++) {
if ((y & 1) == (layout ? 0 : 1)) {
// even numbered rows are left to right
xbegin = 0;
xend = image.width;
xinc = 1;
} else {
// odd numbered rows are right to left
xbegin = image.width - 1;
xend = -1;
xinc = -1;
}
for (x = xbegin; x != xend; x += xinc) {
for (int i=0; i < 8; i++) {
// fetch 8 pixels from the image, 1 for each pin
pixel[i] = image.pixels[x + (y + linesPerPin * i) * image.width];
pixel[i] = colorWiring(pixel[i]);
}
// convert 8 pixels to 24 bytes
for (mask = 0x800000; mask != 0; mask >>= 1) {
byte b = 0;
for (int i=0; i < 8; i++) {
if ((pixel[i] & mask) != 0) b |= (1 << i);
}
data[offset++] = b;
}
}
}
}
// translate the 24 bit color from RGB to the actual
// order used by the LED wiring. GRB is the most common.
int colorWiring(int c) {
// return c; // RGB
return ((c & 0xFF0000) >> 8) | ((c & 0x00FF00) << 8) | (c & 0x0000FF); // GRB - most common wiring
}
// ask a Teensy board for its LED configuration, and set up the info for it.
void serialConfigure(String portName) {
if (numPorts >= maxPorts) {
println("too many serial ports, please increase maxPorts");
errorCount++;
return;
}
try {
ledSerial[numPorts] = new Serial(this, portName);
if (ledSerial[numPorts] == null) throw new NullPointerException();
ledSerial[numPorts].write('?');
} catch (Throwable e) {
println("Serial port " + portName + " does not exist or is non-functional");
errorCount++;
return;
}
delay(50);
String line = ledSerial[numPorts].readStringUntil(10);
if (line == null) {
println("Serial port " + portName + " is not responding.");
println("Is it really a Teensy 3.0 running VideoDisplay?");
errorCount++;
return;
}
String param[] = line.split(",");
if (param.length != 12) {
println("Error: port " + portName + " did not respond to LED config query");
errorCount++;
return;
}
// only store the info and increase numPorts if Teensy responds properly
ledImage[numPorts] = new PImage(Integer.parseInt(param[0]), Integer.parseInt(param[1]), RGB);
ledArea[numPorts] = new Rectangle(Integer.parseInt(param[5]), Integer.parseInt(param[6]),
Integer.parseInt(param[7]), Integer.parseInt(param[8]));
ledLayout[numPorts] = (Integer.parseInt(param[5]) == 0);
numPorts++;
}
// draw runs every time the screen is redrawn - show the movie...
void draw() {
//get screen shot
screenShot = getScreen();
//contrast - this probably doesn't need to be bigger than 5 or 10
float contrast = 5f * ( sdr.getValueI() / (float)width); //value should go from 0 to 5
println("contrast: " + contrast);
//brightness - this is additive and we want to make it both brighter and darker so -128 to +128
float bright = 255 * ( sdr.getValueI() / (float)width - 0.5); //value should go from -128 to +128
println("bright: " + contrast);
//248,136 top left corner of Youtube video
screenShot_crop.copy(screenShot,280,330,600,80,0,0,600,80);
// screenShot.blend(0, 0, 600, 80, 0, 0, 600, 80, DARKEST);
//set contrast with value from slider
ContrastAndBrightness(screenShot_crop,screenShot_en, contrast,bright);
image(screenShot_en,0,0);
for (int i=0; i < numPorts; i++) {
// copy a portion of the movie's image to the LED image
int xoffset = percentage(screenShot_en.width, ledArea[i].x);
int yoffset = percentage(screenShot_en.height, ledArea[i].y);
int xwidth = percentage(screenShot_en.width, ledArea[i].width);
int yheight = percentage(screenShot_en.height, ledArea[i].height);
ledImage[i].copy(screenShot_en, xoffset, yoffset, xwidth, yheight,
0, 0, ledImage[i].width, ledImage[i].height);
// convert the LED image to raw data
byte[] ledData = new byte[(ledImage[i].width * ledImage[i].height * 3) + 3];
image2data(ledImage[i], ledData, ledLayout[i]);
if (i == 0) {
ledData[0] = '*'; // first Teensy is the frame sync master
int usec = (int)((1000000.0 / framerate) * 0.75);
ledData[1] = (byte)(usec); // request the frame sync pulse
ledData[2] = (byte)(usec >> 8); // at 75% of the frame time
} else {
ledData[0] = '%'; // others sync to the master board
ledData[1] = 0;
ledData[2] = 0;
}
// send the raw data to the LEDs :-)
ledSerial[i].write(ledData);
}
}
// scale a number by a percentage, from 0 to 100
int percentage(int num, int percent) {
double mult = percentageFloat(percent);
double output = num * mult;
return (int)output;
}
// scale a number by the inverse of a percentage, from 0 to 100
int percentageInverse(int num, int percent) {
double div = percentageFloat(percent);
double output = num / div;
return (int)output;
}
// convert an integer from 0 to 100 to a float percentage
// from 0.0 to 1.0. Special cases for 1/3, 1/6, 1/7, etc
// are handled automatically to fix integer rounding.
double percentageFloat(int percent) {
if (percent == 33) return 1.0 / 3.0;
if (percent == 17) return 1.0 / 6.0;
if (percent == 14) return 1.0 / 7.0;
if (percent == 13) return 1.0 / 8.0;
if (percent == 11) return 1.0 / 9.0;
if (percent == 9) return 1.0 / 11.0;
if (percent == 8) return 1.0 / 12.0;
return (double)percent / 100.0;
}
PImage getScreen() {
GraphicsEnvironment ge = GraphicsEnvironment.getLocalGraphicsEnvironment();
GraphicsDevice[] gs = ge.getScreenDevices();
DisplayMode mode = gs[0].getDisplayMode();
Rectangle bounds = new Rectangle(0, 0, mode.getWidth(), mode.getHeight());
BufferedImage desktop = new BufferedImage(mode.getWidth(), mode.getHeight(), BufferedImage.TYPE_INT_RGB);
try {
desktop = new Robot(gs[0]).createScreenCapture(bounds);
}
catch(AWTException e) {
System.err.println("Screen capture failed.");
}
return (new PImage(desktop));
}
void ContrastAndBrightness(PImage input, PImage output,float cont,float bright)
{
int w = input.width;
int h = input.height;
//our assumption is the image sizes are the same
//so test this here and if it's not true just return with a warning
if(w != output.width || h != output.height)
{
println("error: image dimensions must agree");
return;
}
//this is required before manipulating the image pixels directly
input.loadPixels();
output.loadPixels();
//loop through all pixels in the image
for(int i = 0; i < w*h; i++)
{
//get color values from the current pixel (which are stored as a list of type 'color')
color inColor = input.pixels[i];
//here the much faster version (uses bit-shifting) - uncomment to try
int r = (inColor >> 16) & 0xFF; //like calling the function red(), but faster
int g = (inColor >> 8) & 0xFF;
int b = inColor & 0xFF;
//apply contrast (multiplication) and brightness (addition)
r = (int)(r * cont + bright); //floating point aritmetic so convert back to int with a cast (i.e. '(int)');
g = (int)(g * cont + bright);
b = (int)(b * cont + bright);
//slow but absolutely essential - check that we don't overflow (i.e. r,g and b must be in the range of 0 to 255)
//to explain: this nest two statements, sperately it would be r = r < 0 ? 0 : r; and r = r > 255 ? 255 : 0;
//you can also do this with if statements and it would do the same just take up more space
r = r < 0 ? 0 : r > 255 ? 255 : r;
g = g < 0 ? 0 : g > 255 ? 255 : g;
b = b < 0 ? 0 : b > 255 ? 255 : b;
//and again in reverse for illustration - calling the color function is slow so use the bit-shifting version below
output.pixels[i] = color(r ,g,b);
//output.pixels[i]= 0xff000000 | (r << 16) | (g << 8) | b; //this does the same but faster
}
//so that we can display the new image we must call this for each image
input.updatePixels();
output.updatePixels();
}