Split to layers by color areas in Photoshop automatically

Question

How can I convert an image like this (known as an Object ID pass or Clown Pass in 3d) to a layered file where every layer would be a color blob (using Photoshop only)?

I know how to do that manually, but I'm interested in an automatic algorithm.

Maybe I somehow can isolate colors and create BW-masks based on those to use them later in a script?

My current idea goes like this:

1. get a color from a left-top corner (black in this case) of the original layer (OL)
2. use Color Range to select this color
3. create a new layer via cut, hide the new layer
4. select the OL, create a selection using its transparency
5. create a path based on this selection
6. find a point inside this path and get a color from this point
7. repeat from 2) with the new color

But this seems cumbersome and time consuming.

Answer 1

If there are 256 or fewer regions, then this can be accomplished using an indexed colour duplicate of the image to define the matte regions.

Here's a script which attempts to automate this on the current document:

#target photoshop

var ChannelIDs = {
    RED: "Rd  ",
    GREEN: "Grn ",
    BLUE: "Bl  ",
    RGB: "RGB "
};

function isSelectionEmpty(doc) {
    try {
        return (doc.selection.bounds) ? false : true;
    } catch (err) {
        return true;
    }
}

function createChannelFromSelection(doc, channelName) {
    var chan = doc.channels.add();
    chan.name = channelName;
    chan.kind = ChannelType.SELECTEDAREA;
    doc.selection.store(chan, SelectionType.REPLACE);
}

function cutToLayer() { // "Layer via cut"
    executeAction(charIDToTypeID("CtTL"), undefined, DialogModes.NO);
}

function createSnapshot(snapshotName) {
    var makDescriptor = new ActionDescriptor(),
        snapshotAction = new ActionReference(),
        fromRef = new ActionReference();

    snapshotAction.putClass(charIDToTypeID("SnpS"));
    makDescriptor.putReference(charIDToTypeID("null"), snapshotAction);
    fromRef.putProperty(charIDToTypeID("HstS"), charIDToTypeID("CrnH"));
    makDescriptor.putReference(charIDToTypeID("From"), fromRef);

    if (snapshotName) { // Assign snapshot name
        makDescriptor.putString(charIDToTypeID("Nm  "), snapshotName );
        makDescriptor.putEnumerated(charIDToTypeID("Usng"), charIDToTypeID("HstS"), charIDToTypeID("FllD") );
    }

    executeAction(charIDToTypeID("Mk  "), makDescriptor, DialogModes.NO);
}

function restoreSnapshot(snapshotName) {
    if (!snapshotName) { throw new Error("Expected snapshot name"); }

    var selectDescriptor = new ActionDescriptor(),
        snapshotRef = new ActionReference();

    snapshotRef.putName(charIDToTypeID("SnpS"), snapshotName);
    selectDescriptor.putReference(charIDToTypeID("null"), snapshotRef);
    executeAction(charIDToTypeID("slct"), selectDescriptor, DialogModes.NO);
}

function selectChannel(channelId) {
    if (!channelId) { throw new Error("Expected channel ID"); }

    var setDescriptor = new ActionDescriptor(),
        selectRef = new ActionReference(),
        channelRef = new ActionReference(),
        idChnl = charIDToTypeID("Chnl");

    selectRef.putProperty(idChnl, charIDToTypeID("fsel"));
    setDescriptor.putReference(charIDToTypeID("null"), selectRef);
    channelRef.putEnumerated(idChnl, idChnl, charIDToTypeID(channelId));
    setDescriptor.putReference(charIDToTypeID("T   "), channelRef);
    executeAction(charIDToTypeID("setd"), setDescriptor, DialogModes.NO);
};

function loadSelection(docName, channelName) {
    var setDescriptor = new ActionDescriptor(),
        selectRef = new ActionReference(),
        docRef = new ActionReference(),
        idChnl = charIDToTypeID("Chnl");

    selectRef.putProperty(idChnl, charIDToTypeID("fsel"));
    setDescriptor.putReference(charIDToTypeID("null"), selectRef);
    docRef.putName(idChnl, channelName);
    docRef.putName(charIDToTypeID("Dcmn"), docName);
    setDescriptor.putReference(charIDToTypeID("T   "), docRef);
    executeAction(charIDToTypeID("setd"), setDescriptor, DialogModes.NO);
}

function convertToIndexedColour(doc) {
    var opts = new IndexedConversionOptions();

    opts.dither = Dither.NONE;
    opts.forced = ForcedColors.NONE;
    opts.matte = MatteType.NONE;
    opts.palette = Palette.EXACT;
    opts.transparency = false;

    doc.changeMode(ChangeMode.INDEXEDCOLOR, opts);
}

function selectColourTableEntry(i) {
    var actionSet = charIDToTypeID("setd");
    var setDescriptor = new ActionDescriptor();
    var ref = new ActionReference();
    ref.putProperty(charIDToTypeID("Clr "), charIDToTypeID("ClrT"));
    setDescriptor.putReference(charIDToTypeID("null"), ref);

    var colourTable = new ActionList();

    if (i > 0) { // Fill low part of colour table
        for (var n = 0; n < i; ++n) {
            addColourTableEntry(colourTable, 0, 0, 0);
        }
    }

    // Select single entry
    addColourTableEntry(colourTable, 255, 255, 255);

    if (i < 255) { // Fill high part of colour table
        for (var n = i + 1; n < 256; ++n) {
            addColourTableEntry(colourTable, 0, 0, 0);
        }
    }

    setDescriptor.putList(charIDToTypeID("T   "), colourTable);
    executeAction(actionSet, setDescriptor, DialogModes.NO);
}

function addColourTableEntry(colourTable, r, g, b) {
    var entry = new ActionDescriptor();
    entry.putDouble(charIDToTypeID(ChannelIDs.RED), r);
    entry.putDouble(charIDToTypeID(ChannelIDs.GREEN), g);
    entry.putDouble(charIDToTypeID(ChannelIDs.BLUE), b);
    colourTable.putObject(charIDToTypeID("RGBC"), entry);
}

function main() {
    var doc = activeDocument;
    var lyr = doc.activeLayer;

    // Create indexed duplicate
    var dupDoc = doc.duplicate();
    activeDocument = dupDoc;
    convertToIndexedColour(dupDoc);

    var snapshotName = "indexed";
    var channelName = "matte";
    var i;

    // Snapshot the current history state
    createSnapshot(snapshotName);

    for (i = 0; i < 256; ++i) {
        selectColourTableEntry(i);

        // For some reason selecting channels doesn't work in indexed mode?
        dupDoc.changeMode(ChangeMode.RGB);

        // Create new matte from red channel
        selectChannel(ChannelIDs.RED);

        // If there is no selection then we're done
        if (isSelectionEmpty(dupDoc)) { break; }

        createChannelFromSelection(dupDoc, channelName);

        // Use matte to cut new layer in original document
        activeDocument = doc;
        loadSelection(dupDoc.name, channelName);
        cutToLayer();
        doc.activeLayer.name = "Region " + (i + 1);

        // Reset ready for next layer
        doc.activeLayer = lyr;
        activeDocument = dupDoc;
        restoreSnapshot(snapshotName);
    }

    // Clean up
    activeDocument = dupDoc;
    dupDoc.close(SaveOptions.DONOTSAVECHANGES);

    activeDocument = doc;
    doc.activeLayer.remove();

    alert("Found " + i + " region" + (i === 1 ? "" : "s"));
}

main();

If there are more than 256 objects, then some will be merged when converting to indexed colour.

Tested in Photoshop CC

Answer 2

You can create Custom Action, to speed up the process. Start recording and make selection with "Selection by Color Range". Then make Layer Via Copy and return to previous layer and stop recording. Make sure, that you toggle dialog box for Selection. Also you can add shortcut. I've made a sample of such action. Sample action

Answer 3

Thanks everyone for their contribution, in the end I went with my original idea and from what I tested it works fastest and most reliable with antialiased images (aka that have >256 colors).

1. I first put a color sampler (CS) in the left top corner of the image just to have something to work with;
2. using Color Range I load a selection with a low fuziness, do a quickmask in case the selection is half-transparent;
3. I cut the layer using the selection and make it invisible;
4. I select the Original Layer (OL), load a selection based on its transparency, contract and expand it by a set value (to make sure there're no 3-4 px islands);
5. Then I cinvert a selection to path and use this algorythm to find a point within the polygon of the path;
6. I move my CP to that position and sample a new color — this gives me a new color for color range;
7. I repeat the whole thing till there's nothing selectable left on the OL;

Here's some pseudocode:

var colorSample = colorSamplers.add([0,0]);
var shouldContinue = true;

while (shouldContinue)
{
  cutLayer();
}

clean() // deletes color sampler and a path

function cutLayer() {
    colorRange(colorSample.color); // loads a selection based on color from colorSample
    doQuickMask(); // gets rid of large half-transparent areas)
    cut(); // cuts current layer to a new one based on selection from color range
    makeInvisible(); //... and makes it invisible
    selectOriginalLayer(); // selects the original layer
    loadTransparency(); // loads layer non-transparency as selection
    shouldContinue = checkSelection(); // returns false if there's no selection;
    if (!shouldContinue) return; // kills the function
    modifySelection(); // contract and expand selection to get rid of small islands)
    selectionToPath(); // creates a path from selection so that I could get its coordinates
    colorPoint = findPointInsidePolygon(); // returns a point inside the polygon of the selection;
    colorSample.move(colorPoint);
}

and the result: