Programming for Interactive Digital Arts
CS 2, Fall 2023
About Schedule Prev NoteNext Note

18. Image processing

Setting pixel values | Filters | Coloring pixels | Moving pixels | Blending pixels | Neighborhoods

Setting pixel values

There are two equivalent ways to directly set pixel values. The first is the set() function and PImage.set() method. These are the counterparts to get() and PImage.get() — we give coordinates and a color.

[sketch1]
function draw()
{
  for (var y=0; y<height; y++)
    for (var x=0; x<width; x++)
      set(x,y,color(random(255),random(255),random(255)));
}
[applet]screenshot

The other way to set pixel values is to directly manipulate an array of the pixel values. There is a global array pixels[] for the pixels of the window, and each PImage has a field pixels[] with its own pixels. Before using one of these, we must call the loadPixels() function or the PImage.loadPixels() method to fill in the array, else we'll get a null pointer exception. After we're finished, we must call the updatePixels() function or PImage.updatePixels() to store the array else our changes don't take. The pixels[] array is indexed the same way we saw last time with the pieces of the puzzle — rows are put into the array from top to bottom. Thus if the image is 800x600, the array has 480000 elements, with 0-799 from the first row, 800-1599 from the second, etc. To get pixel (x,y), we access array element x+y*width, since we skip width pixels for each row.

Here's the same random colors sketch using the direct pixel approach.

[sketch2]
function draw()
{
  loadPixels();
  for (var y=0; y<height; y++)
    for (var x=0; x<width; x++)
      pixels[x+y*width] = color(random(255),random(255),random(255));
  updatePixels();
}
[applet]screenshot

The direct pixel approach is significantly faster, and is more commonly used, so we'll stick with it. But doing so require writing lots of tedius array indexing code. So here, we will define a set of convenience functions to set/get from screen and images. All sketches in this notes will use these functions, but for the sake of sinplicity, I will hide them from notes (but they are in the linked pde files). Here's another example, with a color gradient (red in x, green in y, and blue in mouse x).

[sketch3]
// convenience function - should be called within loadPixels/updatePixels
function pset(var x, var y, color c) { 
  x = constrain(x,0,width-1);    // make sure we not out of bounds
  y = constrain(y,0,height-1);
  pixels[x+width*y] = c; 
}
color pget(var x, var y) { 
  x = constrain(x,0,width-1);
  y = constrain(y,0,height-1);
  return pixels[x+width*y]; 
}
function pset(PImage i, var x, var y, color c) { 
  x = constrain(x,0,i.width-1);
  y = constrain(y,0,i.height-1);
  i.pixels[x+i.width*y] = c; 
}
color pget(PImage i, var x, var y) { 
  x = constrain(x,0,i.width-1);
  y = constrain(y,0,i.height-1);
  return i.pixels[x+i.width*y]; 
}


function draw()
{
  loadPixels();
  for (var y=0; y<height; y++)
    for (var x=0; x<width; x++)
      pset(x,y,color(x*255.0/width, y*255.0/height, mouseX*255.0/width));
  updatePixels();
}
[applet]screenshot

Filters

Processing provides some functions to modify images. The filter() and PImage.filter() functions apply color modifications such as convert to black and white and grayscale, blur etc. These filtersa re similar to the ones in Photoshop. The following sketch illustrates three of these. Note here how we have to copy the initial image before apply the filter, sine the latter operation changes the image values.

[sketch4]
PImage[] imgs = new PImage[4]; // loaded images
PImage img;                    // current image
var curImg = 0;                // current image

function setup()
{
  createCanvas(400,600);
  
  for(var i = 0; i < imgs.length; i ++) imgs[i] = loadImage("img-"+(i+1)+".jpg");
  img = imgs[curImg];
}

// makes a copy of the current image
PImage makeCopy(PImage src) {
  PImage ret = new PImage(src.width,src.height); // make a image
  ret.copy(src,0,0,src.width,src.height,0,0,ret.width,ret.height); //copy pixels
  return ret;
}

function draw()
{
  // compute the filtered images
  PImage f1 = makeCopy(img); f1.filter(THRESHOLD,map(mouseX,0,width,0,1));
  PImage f2 = makeCopy(img); f2.filter(POSTERIZE,map(mouseX,0,width,100,2));
  PImage f3 = makeCopy(img); f3.filter(BLUR,map(mouseX,0,width,0,10));
  
  // draw them
  background(0);  
  image(img,0,0);
  image(f1,0,height/2);
  image(f2,width/2,0);
  image(f3,width/2,height/2);
}

function keyPressed()
{
  if(key == ' ') {
    curImg = (curImg+1)%imgs.length;
    img = imgs[curImg];
  }
}
[applet]screenshot

If we need to combine two images, Processing provides us with all the layers compositing operations found in Photostop (including their completely weird names). The

blend() and PImage.blend() functions combine pixels from two images, while the PImage.mask() function can be used to set the transparency of one image based on the grayscale values of another, thus allowing for more complex compositing.

[sketch5]
PImage[] imgs = new PImage[4]; // loaded images
PImage img1,img2;              // current images
var curImg1 = 0, curImg2 = 1;  // current images

function setup()
{
  createCanvas(400,600);
  
  for(var i = 0; i < imgs.length; i ++) imgs[i] = loadImage("img-"+(i+1)+".jpg");
  img1 = imgs[curImg1];
  img2 = imgs[curImg2];
}

// makes a copy of the current image
PImage makeCopy(PImage src) {
  PImage ret = new PImage(src.width,src.height); // make a image
  ret.copy(src,0,0,src.width,src.height,0,0,ret.width,ret.height); //copy pixels
  return ret;
}

function draw()
{
  // compute the filtered images
  var w = img1.width; var h = img1.height; 
  PImage f1 = makeCopy(img1); f1.blend(img2,0,0,w,h,0,0,w,h,HARD_LIGHT);
  PImage f2 = makeCopy(img1); f2.blend(img2,0,0,w,h,0,0,w,h,DODGE);
  
  // draw them
  background(0);  
  image(img1,0,0);
  image(img2,width/2,0);
  image(f1,0,height/2);
  image(f2,width/2,height/2);
}

function keyPressed()
{
  if(key == ' ') {
    curImg1 = (curImg1+1)%imgs.length;
    curImg2 = (curImg2+1)%imgs.length;
    img1 = imgs[curImg1];
    img2 = imgs[curImg2];
  } else if(key == 'f') { // swap images
    var a = curImg1;
    curImg1 = curImg2;
    curImg2 = a;
    img1 = imgs[curImg1];
    img2 = imgs[curImg2];
  }
}
[applet]screenshot

Coloring pixels

The real power of processing though is in defining our very own filters. To do so, we will use the low level pixel writing operations introduces above. The following sketch shows three examples of color manipulation.

In filterThreshold, we set pixel values to black or white depending on whether the pixel brughtnss is above a threshold or not.

In filterContrast, we can accentuate the redness/greenness/blueness or the lack thereof for a constrast enhancement effect. Greenberg (10-34) uses a formula that compares the difference between the current value and the middle value (127.5), and scales that difference by a specified factor. For example, if the current value is 130 and the factor is 2, then the new value is 130 + 2*2.5 = 135. Thus the value gets pushed out to one extreme (255) or the other (0), depending on which side of the middle it's on.

In filterInvert, we invert an image by subtracting pixel values from 255. While a standard invert filter works the same for every pixel, once we move to setting pixels ourselves, we can modify it to invert more or less for different pixels. We implement a variation of Greenberg (10-41) in which the inversion varies row by row. In particular, we subtract from 0 for the first row, then 1/height for the second row, then 2/height, etc., taking the absolute value after subtracting. Thus subtracting from 0 does nothing, subtracting from 1 only slightly changes it, ..., till it's completely inverted. Also note that rather than setting the value directly, we blend the filtered value with the original one to soften the effect.

While all the filters here do not have much interaction, Shiffman 15-8 shows how to change the brightness of only the image part below the mouse. Allthe filters shown below vould trivially be adapted for this.

[sketch6]
PImage[] imgs = new PImage[4]; // loaded images
PImage img;                    // current image
var curImg = 0;                // current image

function setup()
{
  createCanvas(400,600);
  
  for(var i = 0; i < imgs.length; i ++) imgs[i] = loadImage("img-"+(i+1)+".jpg");
  img = imgs[curImg];
}

PImage filterThreshold(PImage src) {
  PImage ret = new PImage(src.width,src.height); // make a new image
  var threshold = map(mouseX,0,width,32,200); // threshold
  for (var i=0; i<src.pixels.length; i++) {
    var b = brightness(src.pixels[i]);
    if(b > threshold) ret.pixels[i] = color(255);
    else ret.pixels[i] = color(0);
  }
  ret.updatePixels();
  return ret;
}

// Based on Greenberg 10-34
PImage filterContrast(PImage src) {
  PImage ret = new PImage(src.width,src.height); // make a new image
  var contrast = map(mouseX,0,width,0,.01);  // factor by which to push toward either 0 or 255
  for (var i=0; i<src.pixels.length; i++) {
    color p=src.pixels[i];
    var r=red(p), g=green(p), b=blue(p);
    r = constrain(r+r*contrast*(r-127.5), 0,255);
    g = constrain(g+g*contrast*(g-127.5), 0,255);
    b = constrain(b+b*contrast*(b-127.5), 0,255);
    ret.pixels[i] = color(r,g,b);
  }
  ret.updatePixels();
  return ret;  
}

// Based on Greenberg 10-41
PImage filterInvert(PImage src) {
  PImage ret = new PImage(src.width,src.height); // make a new image
  var invertMag = map(mouseX,0,width,0,1);  // factor by which to invert
  for (var y=0; y<src.height; y++) {
    var invert=y*255.0/img.height;
    for (var x=0; x<src.width; x++) {
      color p=pget(src,x,y);
      var b = brightness(p);
      var nb = b*(1-invertMag)+invertMag*abs(invert-b);
      var r = nb/b;
      pset(ret,x,y,color(red(p)*r,green(p)*r,blue(p)*r));
    }
  }
  ret.updatePixels();
  return ret;  
}

function draw()
{
  // compute the filtered images
  PImage f1 = filterThreshold(img);
  PImage f2 = filterContrast(img);
  PImage f3 = filterInvert(img);
  
  // draw them
  background(0);  
  image(img,0,0);
  image(f1,0,height/2);
  image(f2,width/2,0);
  image(f3,width/2,height/2);
}

function keyPressed()
{
  if(key == ' ') {
    curImg = (curImg+1)%imgs.length;
    img = imgs[curImg];
  }
}
[applet]screenshot

Moving pixels

The filters shown above change the color of each individual pixel. Other common operations are change the spatial location of the pixels in the image.

In filterFlip, we swap the pixels along the X-axis by setting the filtered pixel at position (x,y) to the original one at position (width-1-x,y). The -1 in (width-1-x) comes from the fact that the last x pixel index is width-1, not width.

In filterScramble, we randomly copy pixels from location close to the original one. This gives us a crackly glass.

In filterLens, we obtain a fisheye lens effect. If the pixels are outside the lens, we simply copy from the original locations. If not, we compute the normalized distance between the current location and the lens center. This distance is them applied when funding the location to copy from and has the effect of "pushing" such location far away from lens center (lx,ly).

[sketch7]
PImage[] imgs = new PImage[4]; // loaded images
PImage img;                    // current image
var curImg = 0;                // current image

function setup()
{
  createCanvas(400,600);
  
  for(var i = 0; i < imgs.length; i ++) imgs[i] = loadImage("img-"+(i+1)+".jpg");
  img = imgs[curImg];
}

PImage filterFlip(PImage src) {
  PImage ret = new PImage(src.width,src.height); // make a new image
  for (var y=0; y<src.height; y++) {
    for (var x=0; x<src.width; x++) {
      pset(ret,x,y,pget(src,src.width-1-x,y));
    }
  }
  ret.updatePixels();
  return ret;
}

PImage filterLens(PImage src, var lx, var ly, var lr) {
  PImage ret = new PImage(src.width,src.height); // make a new image
  for (var y=0; y<src.height; y++) {
    for (var x=0; x<src.width; x++) {
      var d = dist(x,y,lx,ly)/lr; // distance to lens center normalized
      if(d >= 1) pset(ret,x,y,pget(src,x,y)); // do nothing outside
      else pset(ret,x,y,pget(src,lx+var((x-lx)*d),ly+var((y-ly)*d)));
    }
  }
  ret.updatePixels();
  return ret;
}

PImage filterScramble(PImage src, var sx) {
  PImage ret = new PImage(src.width,src.height); // make a new image
  for (var y=0; y<src.height; y++) {
    for (var x=0; x<src.width; x++) {
      pset(ret,x,y,pget(src,var(random(x-sx,x+sx)),var(random(y-sx,y+sx))));
    }
  }
  ret.updatePixels();
  return ret;
}

function draw() {
  // compute the filtered images
  PImage f1 = filterFlip(img);
  PImage f2 = filterScramble(img,map(mouseX,0,width,1,10));
  PImage f3 = filterLens(img,mouseX-width/2,mouseY-height/2,50);
  
  // draw them
  background(0);  
  image(img,0,0);
  image(f1,0,height/2);
  image(f2,width/2,0);
  image(f3,width/2,height/2);
}

function keyPressed()
{
  if(key == ' ') {
    curImg = (curImg+1)%imgs.length;
    img = imgs[curImg];
  }
}
[applet]screenshot

Blending pixels

Just like we defined our own filter for a single image, we can define filter to combine two images.

In blendSquares, we selectively display the pixels of an image over another depending on whether those pixels are along image diagonals. the way I came up with the diagonal tests is frankly random. I got the code wrong the first time, while I was trying to do squares, and it came out to be diamonds. I liked it more and left it here. This is the great power of coding in Processing: you can serendepitously find new effects just by playing with it. That can never happen with a canned software.

In blendLookAt, I use a similar formula from the lens example before to show only part of an image over the other.

[sketch8]
PImage[] imgs = new PImage[4]; // loaded images
PImage img1,img2;              // current images
var curImg1 = 0, curImg2 = 1;  // current images

function setup()
{
  createCanvas(400,600);
  
  for(var i = 0; i < imgs.length; i ++) imgs[i] = loadImage("img-"+(i+1)+".jpg");
  img1 = imgs[curImg1];
  img2 = imgs[curImg2];
}

PImage blendSquares(PImage a, PImage b, var s) {
  PImage ret = new PImage(a.width,a.height);
  for(var y = 0; y < ret.height; y ++) {
    for(var x = 0; x < ret.width; x ++) {
      if((x+y)%s==0||(x-y)%s==0) pset(ret,x,y,pget(a,x,y));
      else pset(ret,x,y,pget(b,x,y));
    }
  }
  return ret;
}

PImage blendLookAt(PImage a, PImage b, var lx, var ly, var lr) {
  PImage ret = new PImage(a.width,a.height);
  for(var y = 0; y < ret.height; y ++) {
    for(var x = 0; x < ret.width; x ++) {
      var f = constrain(dist(x,y,lx,ly)/lr,0,1);
      f = f*f; // make it a bit more dramatic
      color ca = pget(a,x,y), cb = pget(b,x,y);
      var rr = red(ca)*f+(1-f)*red(cb);
      var rg = green(ca)*f+(1-f)*green(cb);
      var rb = blue(ca)*f+(1-f)*blue(cb);
      pset(ret,x,y,color(rr,rg,rb));
    }
  }
  return ret;
}

function draw()
{
  // compute the filtered images
  PImage f1 = blendSquares(img1,img2,constrain(var(map(mouseX,0,width/2,1,20)),1,20));
  PImage f2 = blendLookAt(img1,img2,mouseX-width/2,mouseY-height/2,100);
  
  // draw them
  background(0);  
  image(img1,0,0);
  image(img2,width/2,0);
  image(f1,0,height/2);
  image(f2,width/2,height/2);
}

function keyPressed()
{
  if(key == ' ') {
    curImg1 = (curImg1+1)%imgs.length;
    curImg2 = (curImg2+1)%imgs.length;
    img1 = imgs[curImg1];
    img2 = imgs[curImg2];
  } else if(key == 'f') { // swap images
    var a = curImg1;
    curImg1 = curImg2;
    curImg2 = a;
    img1 = imgs[curImg1];
    img2 = imgs[curImg2];
  }
}
[applet]screenshot

Neighborhoods

The examples we coevered so far treat each pixel independently of each other pixel (except for swapping pairs). A whole other class of image processing algorithms deals with the "neighborhood" around a pixel. The neighborhood is often defined as those pixels immediately adjacent to a given pixel, shown in the following picture as green dots.

neigbhorhood

We can write filters that compute a pixel value by performing operation on its neightborhood. This is shown i the next sketch, where the effect of three different operations is shown.

In filterAverage, we compute the average of the pixel's neightborhood. This gives us a blurry appearance, essentially removing out the details.

In filterBlur, we refine on this idea and compute a weighted average, with weights inversely proportional to the pixel distance.

In filterDiff, we implement a kind of edge 'enhancement' where rather than averaging the values near us, we instead subtract them out.

[sketch9]
PImage[] imgs = new PImage[4]; // loaded images
PImage img;                    // current image
var curImg = 0;                // current image

function setup() 
{
  createCanvas(400,600);
  smooth();
  rectMode(CENTER);
  
  for(var i = 0; i < imgs.length; i ++) imgs[i] = loadImage("img-"+(i+1)+".jpg");
  img = imgs[curImg];
}

PImage filterAverage(PImage src, var s) {
  PImage ret = new PImage(src.width,src.height);
  // loop over the pixel I want to write to
  for (var y=0; y<ret.height; y++) {
    for (var x=0; x<ret.width; x++) {
      // add together the values for all the neighbors
      var r=0, g=0, b=0; var w=0;
      for (var dy=-s; dy<=s; dy++) {
        for (var dx=-s; dx<=s; dx++) {
          // our routines make sure we do not go out of bounds
          color p=pget(src,x+dx,y+dy); 
          r += red(p); g += green(p); b += blue(p);
          w++;
        }
      }
      // set the color as the average
      pset(ret,x,y,color(r/w,g/w,b/w));
    }
  }
  ret.updatePixels();
  return ret;
}

PImage filterDiff(PImage src, var s) {
  PImage ret = new PImage(src.width,src.height);
  // loop over the pixel I want to write to
  for (var y=0; y<ret.height; y++) {
    for (var x=0; x<ret.width; x++) {
      // add together the values for all the neighbors
      var r=0, g=0, b=0; var w=0;
      for (var dy=-s; dy<=s; dy++) {
        for (var dx=-s; dx<=s; dx++) {
          // our routines make sure we do not go out of bounds
          color p=pget(src,x+dx,y+dy);
          var k = 0;
          if(dx ==0 && dy == 0) k = (2*s+1)*(2*s+1);
          else k = -1;
          r += k*red(p); g += k*green(p); b += k*blue(p);
          w += k;
        }
      }
      // set the color as the average
      pset(ret,x,y,color(r/w,g/w,b/w));
    }
  }
  ret.updatePixels();
  return ret;
}

PImage filterBlur(PImage src, var s) {
  PImage ret = new PImage(src.width,src.height);
  // loop over the pixel I want to write to
  for (var y=0; y<ret.height; y++) {
    for (var x=0; x<ret.width; x++) {
      // add together the values for all the neighbors
      var r=0, g=0, b=0; var w=0;
      for (var dy=-s; dy<=s; dy++) {
        for (var dx=-s; dx<=s; dx++) {
          // our routines make sure we do not go out of bounds
          color p=pget(src,x+dx,y+dy); 
          var ks = 1-constrain(dist(x,y,x+dx,y+dy)/s,0,1); // count more the central one
          r += ks*red(p); g += ks*green(p); b += ks*blue(p);
          w += ks;
        }
      }
      // set the color as the average
      pset(ret,x,y,color(r/w,g/w,b/w));
    }
  }
  ret.updatePixels();
  return ret;
}

function draw() {
  // compute the filtered images
  PImage f1 = filterAverage(img,var(map(mouseX,0,width,1,8)));
  PImage f2 = filterDiff(img,var(map(mouseX,0,width,1,8)));
  PImage f3 = filterBlur(img,var(map(mouseX,0,width,1,8)));
  
  // draw them
  background(0);  
  image(img,0,0);
  image(f1,0,height/2);
  image(f2,width/2,0);
  image(f3,width/2,height/2);
}

function keyPressed() {
  if(key == ' ') {
    curImg = (curImg+1)%imgs.length;
    img = imgs[curImg];
  }
}
[applet]screenshot