Trying to understand inpainting + OpenCV

[Selur]

Looking at OpenCV-Python:](https://github.com/WolframRhodium/muvsfunc/wiki/OpenCV-Python-for-VapourSynth) I wanted to try to use the sample from here to test the inpainting. Not really knowing what I'm doing, I tried with:

# Imports
import vapoursynth as vs
import os
import ctypes
# Loading Support Files
Dllref = ctypes.windll.LoadLibrary("i:/Hybrid/64bit/vsfilters/Support/libfftw3f-3.dll")
import sys
# getting Vapoursynth core
core = vs.core
# Import scripts folder
scriptPath = 'i:/Hybrid/64bit/vsscripts'
sys.path.insert(0, os.path.abspath(scriptPath))
# Loading Plugins
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/GrainFilter/RemoveGrain/RemoveGrainVS.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/GrainFilter/AddGrain/AddGrain.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/DenoiseFilter/FFT3DFilter/fft3dfilter.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/DenoiseFilter/DFTTest/DFTTest.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/EEDI3m.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/ResizeFilter/nnedi3/vsznedi3.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/libmvtools.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/scenechange.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/fmtconv.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/MiscFilter/MiscFilters/MiscFilters.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/SourceFilter/LSmashSource/vslsmashsource.dll")
core.std.LoadPlugin(path="i:/Hybrid/64bit/vsfilters/Support/libimwri.dll")
# Import scripts
import havsfunc
# source: 'G:\TestClips&Co\files\MPEG-2\ZDF_Logoremoval_Disco 1977-04 077 576p Digi-TVRip - Stormjoe .mkv'
# current color space: YUV420P8, bit depth: 8, resolution: 720x576, fps: 25, color matrix: 470bg, yuv luminance scale: limited, scanorder: top field first
# Loading G:\TestClips&Co\files\MPEG-2\ZDF_Logoremoval_Disco 1977-04 077 576p Digi-TVRip - Stormjoe .mkv using LWLibavSource
clip = core.lsmas.LWLibavSource(source="G:/TestClips&Co/files/MPEG-2/ZDF_Logoremoval_Disco 1977-04 077 576p Digi-TVRip - Stormjoe .mkv", format="YUV420P8", stream_index=0, cache=0, prefer_hw=0)
# Setting color matrix to 470bg.
clip = core.std.SetFrameProps(clip, _Matrix=5)
clip = clip if not core.text.FrameProps(clip,'_Transfer') else core.std.SetFrameProps(clip, _Transfer=5)
clip = clip if not core.text.FrameProps(clip,'_Primaries') else core.std.SetFrameProps(clip, _Primaries=5)
# Setting color range to TV (limited) range.
clip = core.std.SetFrameProp(clip=clip, prop="_ColorRange", intval=1)
# making sure frame rate is set to 25
clip = core.std.AssumeFPS(clip=clip, fpsnum=25, fpsden=1)
# setting field order to what QTGMC should assume (top field first)
clip = core.std.SetFrameProp(clip=clip, prop="_FieldBased", intval=2)
# Deinterlacing using QTGMC
clip = havsfunc.QTGMC(Input=clip, Preset="Fast", TFF=True) # new fps: 25
# make sure content is preceived as frame based
clip = core.std.SetFieldBased(clip, 0)
clip = clip[::2]

# resize to square pixel
clip = core.resize.Bicubic(clip=clip, width=768, height=576)

input = clip

##### INPAINTING CODE START

import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    dst = np.empty_like(img)
    cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)
    return dst

mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.std.BinarizeMask(mask,threshold=16)
mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")

inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

# set output frame rate to 25fps
clip = core.std.AssumeFPS(clip=clip, fpsnum=25, fpsden=1)

clip = core.std.StackHorizontal([input, clip])
# Output
clip.set_output()

It's not crashing, but the output: isn't what I was hoping for. From the looks of it, there is some color space issue.

-> Would be nice if someone could post an example with the given clip&logo how to use inpainting. :)

Thanks!

[WolframRhodium]

Thanks, the wiki has been corrected.

[Selur]

Thanks for looking into it, I changed my code to:

##### INPAINTING CODE START
mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.std.BinarizeMask(mask,threshold=16)
mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")

import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    return cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)

inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

Now, the output is not broken, but now the filter doesn't seem to do anything.

Cu Selur

[WolframRhodium]

clip = core.resize.Bicubic(clip, format=vs.RGB24, matrix_in_s="470bg")

mask = core.std.ShufflePlanes(mask, [0], vs.GRAY)

inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS, input_per_plane=False, output_per_plane=False)

[Selur]

Using:

##### INPAINTING CODE START

# convert clip to RGB
clip = core.resize.Bicubic(clip, format=vs.RGB24, matrix_in_s="470bg")

# build mask
mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.std.BinarizeMask(mask,threshold=16)
mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
mask = core.std.ShufflePlanes(mask, [0], vs.GRAY)

# apply inpainting
import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    return cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)
inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS, input_per_plane=False, output_per_plane=False)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

It does not do anything (aside from converting to RGB).

Seems like something is missing.

[Selur]

Got it! The mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")' needs to be removed! :)

Using:

##### INPAINTING CODE START

# convert clip to RGB
clip = core.resize.Bicubic(clip, format=vs.RGB24, matrix_in_s="470bg")

# build mask
mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.std.BinarizeMask(mask,threshold=16)
mask = core.std.ShufflePlanes(mask, [0], vs.GRAY)

# apply inpainting
import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    return cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)
inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS, input_per_plane=False, output_per_plane=False)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

it works:

[Selur]

Related question: Why the clip = core.std.MaskedMerge(clip, inpainting, mask) call? Comparing inpainting with clip = core.std.MaskedMerge(clip, inpainting, mask) there doesn't seem to be a difference.

[WolframRhodium]

Yes, MaskedMerge could be removed.

[Selur]

Okay, thanks! :) Now I just need to get it working better. :)

Thanks for the help. :)

[WolframRhodium]

Anyway, for your original script, it should be

mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
mask = core.std.BinarizeMask(mask,threshold=128)

# and then
# inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS)

[Selur]

I'm not getting it, using:

##### INPAINTING CODE START

import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    dst = np.empty_like(img)
    cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)
    return dst

mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
mask = core.std.BinarizeMask(mask,threshold=128)

inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

in the original script still gives me the color mess.

Same when using the updated inpaint_core:

##### INPAINTING CODE START

import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    return cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)

mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
mask = core.std.BinarizeMask(mask,threshold=128)

inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

[WolframRhodium]

##### INPAINTING CODE START

import cv2
import muvsfunc_numpy as mufnp
import numpy as np

def inpaint_core(img, mask, radius=1, flags=cv2.INPAINT_NS):
    return cv2.inpaint(img, mask, inpaintRadius=radius, flags=flags)

mask = core.imwri.Read(["C:/Users/Selur/Desktop/logo.png"])
mask = core.resize.Bicubic(clip=mask, format=vs.YUV420P8, matrix_s="470bg", range_s="limited", dither_type="error_diffusion")
mask = core.std.BinarizeMask(mask,threshold=128)

inpainting = mufnp.numpy_process([clip, mask], inpaint_core, radius=1, flags=cv2.INPAINT_NS)
clip = core.std.MaskedMerge(clip, inpainting, mask)

##### INPAINTING CODE END

[Selur]

That's the same code I posted,...

[WolframRhodium]

Then just use the rgb approach.

[Selur]

okay, I'll stick with the RGB variant.