Trouble with colors of segments

niccolo99mandelli commented 1 year ago

Hi, after starting the segmentation process, I need to retrieve the colors assigned to the labels in the segmented image. From the code, it seems that the color intensity of the labels is given by the attribute segments_mean. However, this attribute returns data of type :Real. Is there a way to convert the :Real value to its corresponding RGB :Colorant value ?

Thanks in advance, NM

ashwani-rathee commented 1 year ago

Hey Niccolo, Can you please provide a MWE?

niccolo99mandelli commented 1 year ago

Yes, of course. This is my main code :

link = joinpath(@__DIR__, "..", "input_example_demo", "slideExample1", "SlideExample_mini_1.tif")

# load slide
println("LOAD SLIDE ...")
svs_image = read(link)
img = ImageMagick.load_(svs_image)
bw = Gray.(img) .> 0.15
dist = 1 .- distance_transform(feature_transform(bw))
markers = label_components(dist .< -0.3)

# watershed
println("APPLY SEGMENTATION ...")
segments = watershed(dist, markers)

# build segmented slide
println("BUILD SEGMENTED SLIDE ...")
labels = labels_map(segments)
colored_labels = IndirectArray(labels, distinguishable_colors(maximum(labels)))
masked_colored_labels = colored_labels .* (1 .- bw)

# build graph
println("BUILD GRAPH ...")
weight_fn(i,j) = euclidean(segment_pixel_count(segments,i), segment_pixel_count(segments,j))
df_labels = DataFrame()
G, vert_map, df_labels = region_adjacency_graph(segments, weight_fn)

I have modified the function region_adjacency_graph in your package with the following :

function region_adjacency_graph(s::SegmentedImage, weight_fn::Function)

    function neighbor_regions!(df_cartesian_indices::AbstractArray, G::SimpleWeightedGraph, visited::AbstractArray, s::SegmentedImage, I::CartesianIndex)
        # n = Set{Int} - visited = Array - s = segmented image - p = CartesianIndex which define neighbors
        # R contains each possible index in s
        R = CartesianIndices(axes(s.image_indexmap))
        # I1 contains a Vector of only 1 with dimension equal to visited
        I1 = _oneunit(CartesianIndex{ndims(visited)})
        # Ibegin and Iend contains the first and last index of R
        Ibegin, Iend = first(R), last(R)
        # t is only a empty Vector with dimension equal to visited
        t = Vector{CartesianIndex{ndims(visited)}}()
        # add index I to Vector t
        push!(t, I)
        while !isempty(t)
            # Extract last element of t and save it in temp
            temp = pop!(t)
            # set index temp to true
            visited[temp] = true
            # _colon build an object CartesianIndices which include all the index from I to J (range) :
            for J in _colon(max(Ibegin, temp-I1), min(Iend, temp+I1))
                if s.image_indexmap[temp] != s.image_indexmap[J]
                    if !Graphs.has_edge(G, vert_map[s.image_indexmap[I]], vert_map[s.image_indexmap[J]])
                        Graphs.add_edge!(G, vert_map[s.image_indexmap[I]], vert_map[s.image_indexmap[J]], weight_fn(s.image_indexmap[I], s.image_indexmap[J]))
                    end
                elseif !visited[J]
                    # If they are equal, I place them in t, so that,
                    # as long as t is not empty, I can explore all the neighbors
                    # that have the same color.
                    push!(t,J)
                end
            end
        end
    end
    # Start
    visited  = fill(false, axes(s.image_indexmap))                           # Array to mark the pixels that are already visited
    G        = SimpleWeightedGraph()                                         # The region_adjacency_graph
    vert_map = Dict{Int,Int}()                                               # Map that stores (label, vertex) pairs
    # Build object for label (vertex) dataframe
    df_label = DataFrame()
    df_cartesian_indices = []
    df_color_indices = []
    # add vertices to graph
    Graphs.add_vertices!(G,length(s.segment_labels))
    # setup `vert_map`
    for (i,l) in enumerate(s.segment_labels)
        vert_map[l] = i
    end
    # add edges to graph
    # For each CartesianIndices in s where the image_indexmap represent the image wich is a Multidimensional Array
    # The index of s.image_indexmap represent the pixel position in the segmented image
    # The value of s.image_indexmap represent the pixel color in the segmented image
    for p in CartesianIndices(axes(s.image_indexmap))
        # check if p of the segmented image s is not visited
        if !visited[p]
            push!(df_cartesian_indices, p)
            # n = Set{Int16}()
            # Call neighbor_regions where :
            # n = Set{Int} - visited = Array - s = segmented image - p = CartesianIndex which define neighbors
            try
                neighbor_regions!(df_cartesian_indices, G, visited, s, p)
            catch oom
                if isa(oom, OutOfMemoryError)
                    # n = Set{Int}()
                    GC.gc()
                    println(">>> OOM")
                    exit()
                end
            end
        end
    end

    for i in s.segment_labels
        push!(df_color_indices, s.segment_means[i])
    end
    df_label.label = s.segment_labels
    df_label.position_label = df_cartesian_indices
    df_label.color_label = df_color_indices
    G, vert_map, df_label
end

The main difference lies in creating a DataFrame containing label information (vertices). The DataFrame consists of three columns: the index (:Int), the position (:CartesianIndex), and the color of the label (:Real). As mentioned earlier, I obtain the color from the data: segment_means::Dict{Int,U} where U<:Union{Colorant,Real} ... Unfortunately, with the value :Real, I am unable to obtain the RGB value.

niccolo99mandelli commented 1 year ago

Do you have any updates for me?

ashwani-rathee commented 1 year ago

Sorry for the delayed reply, your issue starts right at watershed() which return Float64 in segments.segment_means, because of the input. watershed also restricts input to numbers or Gray. Any specific reasons to use watershed?

niccolo99mandelli commented 1 year ago

Yes, I am using watershed() to segment histopathological images. My request arises from the need to print, once the segmentation is completed, the segmented image with the overlaid graph generated from cell segmentation. Each segmented cell corresponds to its respective node in the graph. However, to have an effective visual representation, I would color the nodes with the colors generated by watershed() for each segment, making it more meaningful... But now I have a spontaneous question, what does the value s.segment_mean refer to? What is meant by "segment mean intensity"?

ashwani-rathee commented 1 year ago

s.segment_means returns a dict with Int->Color, Those int are part of s.image_indexmap which let's say is 512*512 where each indice is assigned a value based on which segment it belongs to and each of those values have a corresponding key->value pair in s.segment_means.

Let me share an example:

using Images, TestImages

img = imresize(testimage("mandrill"), (200,200))

julia> seg = unseeded_region_growing(img, 0.3) 
Segmented Image with:
  labels map: 200×200 Matrix{Int64}
  number of labels: 14

julia> seg.image_indexmap # each pixel has a label after segmentation
200×200 Matrix{Int64}:
 1  1  1  1  1  1  4  1  1  1  4  4  …  7  7  7  6  6  7  7  7  7  7  7   7
 2  1  1  1  1  1  1  1  1  1  4  1     7  7  7  7  7  7  7  7  7  7  7   1
 1  1  1  1  1  1  1  1  1  1  4  4     7  7  1  6  7  7  7  7  7  7  1  13
 1  1  1  1  1  4  1  1  1  4  4  4     7  7  6  7  7  6  7  7  7  1  1   1
 1  1  1  1  4  1  1  1  4  4  4  4     7  1  7  7  7  7  7  1  1  7  1   1
 1  1  1  4  1  1  1  4  4  4  1  1  …  7  7  7  7  7  7  7  7  7  7  7   7
 1  1  1  1  1  1  1  1  4  1  1  4     7  7  7  6  7  7  1  7  7  1  7   7
 1  1  1  1  1  1  4  1  1  1  1  4     7  7  7  1  1  1  7  7  7  7  7   7
 1  1  1  1  1  1  1  1  1  4  4  1     7  7  7  7  7  7  7  7  7  7  7   7
 1  1  1  1  1  1  1  4  4  1  4  4     7  7  7  7  6  7  7  7  7  7  7   7
 1  1  1  1  4  1  1  1  1  4  1  1  …  7  6  6  7  7  6  7  7  7  7  7   7
 1  1  1  1  4  1  4  1  1  4  1  1     6  7  7  7  7  4  7  7  7  1  1   7
 1  1  1  1  1  1  1  1  1  4  1  1     6  6  7  7  7  4  7  7  7  7  7   7
 ⋮              ⋮              ⋮     ⋱        ⋮              ⋮           
 2  2  2  2  2  2  2  2  2  2  2  2     2  2  2  2  2  2  2  2  2  9  9   2
 2  2  2  2  2  2  2  2  2  2  2  2     2  2  2  2  2  2  2  2  2  2  2   2
 2  2  2  2  2  2  2  2  2  2  2  2  …  2  2  2  2  2  2  2  2  2  1  2   2
 2  2  2  2  2  2  2  2  2  2  2  2     1  2  2  2  2  2  2  1  1  1  1   2
 2  2  2  2  2  2  2  2  2  2  2  2     1  1  2  2  1  2  2  1  1  1  1   1
 2  2  2  2  2  2  2  2  2  2  2  2     1  1  2  2  1  1  2  1  1  1  1   1
 2  2  2  2  2  2  2  2  2  2  2  2     1  2  2  2  2  1  1  1  1  1  1   1
 2  2  2  2  2  2  2  2  2  2  2  2  …  2  2  2  2  2  1  1  1  1  1  1   1
 2  2  2  2  2  2  2  2  2  2  2  2     2  2  1  1  1  1  1  1  1  1  1   1
 2  2  2  2  2  2  2  2  2  2  2  2     2  2  1  1  1  1  1  1  1  1  1   1
 2  2  2  2  2  2  2  2  2  2  2  2     2  1  1  1  1  1  1  1  1  1  1   1
 2  1  1  2  2  2  2  2  2  2  2  2     1  1  1  1  1  1  1  1  1  1  1   1

julia> seg.segment_labels # labels that are there
14-element Vector{Int64}: 
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14

julia> seg.segment_means # mean color of each segment
Dict{Int64, RGB{Float32}} with 14 entries:
  5  => RGB{Float32}(0.914423,0.334988,0.235096)
  12 => RGB{Float32}(0.79094,0.553753,0.0896552)
  8  => RGB{Float32}(0.550147,0.740876,0.879216)
  1  => RGB{Float32}(0.324067,0.33606,0.266216)
  6  => RGB{Float32}(0.697002,0.718306,0.685223)
  11 => RGB{Float32}(0.567955,0.341948,0.185561)
  9  => RGB{Float32}(0.419085,0.54952,0.631892)
  14 => RGB{Float32}(0.241629,0.335143,0.492609)
  3  => RGB{Float32}(0.134497,0.115769,0.128431)
  7  => RGB{Float32}(0.45431,0.515487,0.437236)
  4  => RGB{Float32}(0.607278,0.603537,0.368445)
  13 => RGB{Float32}(0.74053,0.766818,0.562206)
  2  => RGB{Float32}(0.600762,0.581384,0.44312)
  10 => RGB{Float32}(0.824364,0.520214,0.579298)

julia> seg.segment_pixel_count # number of pixels in each label
Dict{Int64, Int64} with 14 entries:
  5  => 4584
  12 => 29
  8  => 5093
  1  => 9885
  6  => 2268
  11 => 239
  9  => 1005
  14 => 13
  3  => 236
  7  => 6353
  4  => 1458
  13 => 91
  2  => 8384
  10 => 362

julia> for i in eachindex(img)
    # @info i seg.image_indexmap[i] seg.segment_means[seg.image_indexmap[i]]
   img[i] = seg.segment_means[seg.image_indexmap[i]]
end

Above code will change the original color to color of segment's mean for a certain label

JuliaImages / ImageSegmentation.jl

Trouble with colors of segments #96