o my own question (very similar: I need to flow vertically but constrain width horizontally), I got a quick example sort of working and maybe it gets you started with a custom layout manager:

import java.awt.Component;
import java.awt.Container;
import java.awt.Dimension;
import java.awt.LayoutManager2;
import java.util.Random;
import java.util.Set;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.JPanel;
import com.google.common.collect.Sets;

public class VerticalFlowLayout implements LayoutManager2
{
    final private Set<Component> components = Sets.newLinkedHashSet();
    private int hgap = 0;
    private int vgap = 0;

    public void setHGap(int hgap) { this.hgap = hgap; }
    public void setVGap(int vgap) { this.vgap = vgap; }

    @Override public void addLayoutComponent(Component comp, Object constraints) {
        this.components.add(comp);
    }

    /* these 3 methods need to be overridden properly */
    @Override public float getLayoutAlignmentX(Container target) {
        // TODO Auto-generated method stub
        return 0;
    }

    @Override public float getLayoutAlignmentY(Container target) {
        // TODO Auto-generated method stub
        return 0;
    }

    @Override public void invalidateLayout(Container target) {
        // TODO Auto-generated method stub

    }

    @Override public void addLayoutComponent(String name, Component comp) {
        this.components.add(comp);
    }

    @Override public void layoutContainer(Container parent) {
        int x = 0;
        int y = 0;
        int columnWidth = 0;
        for (Component c : this.components)
        {
            if (c.isVisible())
            {
                Dimension d = c.getPreferredSize();
                columnWidth = Math.max(columnWidth, d.width);
                if (y+d.height > parent.getHeight())
                {
                    x += columnWidth + this.hgap;
                    y = 0;
                }
                c.setBounds(x, y, d.width, d.height);
                y += d.height +