optisynapsis
commented
5 months ago In lieu of submitting a fix, I'll just outline the modifications here 👍 The changes exclusively exist within the render_interpolation() function.
def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, freeu_args, root):
class ExtendedDict(dict):
def get_first_none(self):
"""Returns the first key whose value is None."""
try:
return next(k for k, v in self.items() if v is None)
except StopIteration:
return None
def get_last_not_none(self):
"""Returns the last key whose value is not None."""
return max(k for k, v in self.items() if v is not None)
# use parseq if manifest is provided
parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args, freeu_args)
# expand key frame strings to values
keys = DeformAnimKeys(anim_args) if not parseq_adapter.use_parseq else parseq_adapter.anim_keys
frame_idx = 0
last_preview_frame = 0
frame_idx_map = ExtendedDict({i: None for i in range(anim_args.max_frames)})
# Resume from timestring
if anim_args.resume_from_timestring:
print(f"\033[36mAttempting to resume interpolation render")
root.timestring = anim_args.resume_timestring
print(f"\033[36mTimestring:\033[0m {root.timestring}")
""" MOD: OLDCODE :
# count previous frames
frame_count = 0
for item in os.listdir(args.outdir):
# don't count txt files or mp4 files
if ".txt" in item or ".mp4" in item:
pass
else:
filename = item.split("_")
# other image file types may be supported in the future,
# so we just count files containing timestring
# that don't contain the depth keyword (depth maps are saved in same folder)
if root.timestring in filename and "depth" not in filename:
frame_count += 1
frame_idx = frame_count
last_preview_frame = frame_count
ENDMOD
"""
# Loop over every item in the output directory specified by 'args.outdir'
for item in os.listdir(args.outdir):
if item.endswith('.png'):
filename = item.split('_')
if len(filename) > 1 and filename[-1] != '':
idx_str = filename[1].zfill(9)
idx = int(os.path.splitext(idx_str)[0])
frame_idx_map[idx] = idx
print(f"Frame #{idx} has been rendered.")
print(f"\033[36mResuming:\033[0m Next frame: {frame_idx_map.get_first_none()} ")
# create output folder for the batch
os.makedirs(args.outdir, exist_ok=True)
print(f"Saving interpolation animation frames to {args.outdir}")
# save settings.txt file for the current run
save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, freeu_args, video_args, root)
# Compute interpolated prompts
if parseq_adapter.manages_prompts():
print("Parseq prompts are assumed to already be interpolated - not doing any additional prompt interpolation")
prompt_series = keys.prompts
else:
print("Generating interpolated prompts for all frames")
prompt_series = interpolate_prompts(root.animation_prompts, anim_args.max_frames)
state.job_count = anim_args.max_frames
#frame_idx = 0
#last_preview_frame = 0
# INTERPOLATION MODE
def allocate_next_frame():
"""Allocates the next available frame for rendering."""
free_slot = frame_idx_map.get_first_none()
frame_idx_map[free_slot] = True
return free_slot
frame_idx = allocate_next_frame()
while frame_idx is not None:
# print data to cli
prompt_to_print = get_parsed_value(prompt_series[frame_idx].strip(), frame_idx, anim_args.max_frames)
if prompt_to_print.endswith("--neg"):
prompt_to_print = prompt_to_print[:-5]
print(f"\033[36mInterpolation frame: \033[0m{frame_idx}/{anim_args.max_frames} ")
print(f"\033[32mSeed: \033[0m{args.seed}")
print(f"\033[35mPrompt: \033[0m{prompt_to_print}")
state.job = f"frame {frame_idx + 1}/{anim_args.max_frames}"
state.job_no = frame_idx + 1
if state.interrupted:
break
if state.skipped:
print("\n** PAUSED **")
state.skipped = False
while not state.skipped:
time.sleep(0.1)
print("** RESUMING **")
# grab inputs for current frame generation
args.prompt = prompt_to_print
args.scale = keys.cfg_scale_schedule_series[frame_idx]
args.pix2pix_img_cfg_scale = keys.pix2pix_img_cfg_scale_series[frame_idx]
scheduled_sampler_name = keys.sampler_schedule_series[frame_idx].casefold() if anim_args.enable_sampler_scheduling and keys.sampler_schedule_series[frame_idx] is not None else None
args.steps = int(keys.steps_schedule_series[frame_idx]) if anim_args.enable_steps_scheduling and keys.steps_schedule_series[frame_idx] is not None else args.steps
scheduled_clipskip = int(keys.clipskip_schedule_series[frame_idx]) if anim_args.enable_clipskip_scheduling and keys.clipskip_schedule_series[frame_idx] is not None else None
args.checkpoint = keys.checkpoint_schedule_series[frame_idx] if anim_args.enable_checkpoint_scheduling else None
if anim_args.enable_subseed_scheduling:
root.subseed = int(keys.subseed_schedule_series[frame_idx])
root.subseed_strength = keys.subseed_strength_schedule_series[frame_idx]
else:
root.subseed, root.subseed_strength = keys.subseed_schedule_series[frame_idx], keys.subseed_strength_schedule_series[frame_idx]
if parseq_adapter.manages_seed():
anim_args.enable_subseed_scheduling = True
root.subseed, root.subseed_strength = int(keys.subseed_schedule_series[frame_idx]), keys.subseed_strength_schedule_series[frame_idx]
args.seed = int(keys.seed_schedule_series[frame_idx]) if (args.seed_behavior == 'schedule' or parseq_adapter.manages_seed()) else args.seed
#opts.data["CLIP_stop_at_last_layers"] = scheduled_clipskip if scheduled_clipskip is not None else opts.data["CLIP_stop_at_last_layers"]
image = generate(args, keys, anim_args, loop_args, controlnet_args, freeu_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name)
filename = f"{root.timestring}_{frame_idx:09}.png"
save_image(image, 'PIL', filename, args, video_args, root)
##MOD
filenamecfgscale = os.path.join(args.outdir,f"{root.timestring}_{frame_idx:09}_CFG_{args.scale}_step_{args.steps}.txt")
with open(filenamecfgscale, "w") as f:
pass
##ENDMOD
state.current_image = image
if args.seed_behavior != 'schedule':
args.seed = next_seed(args, root)
last_preview_frame = render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame)
frame_idx = allocate_next_frame()
if frame_idx is not None:
frame_idx_map[frame_idx] = frame_idx
This allows for missing frames to be rendered in between the start and end of the sequence : for example, if I previously rendered a sequence where cfg scale interpolates from 0: (1), 9: (10), I could programmatically rename the files with an increment of 2, change the cfg settings to 0: (1), 19: (10), and render only the frames between the previously rendered sequence. This is an edge case, but useful for when variance is unpredictable.
Line 163 is commented out in order for the resume function to work correctly.
I'd like to work on a more elegant solution, but this at least fixes the bug where resuming an interpolation sequence doesn't detect previously rendered frames.
If anybody could advise on how to not just blurt out a bunch of code and do this more respectably that'd also be appreciated :)
What happened and why was it ungood?
I have my own modified version of render_modes.py which I adapted for personal use,
but it seems in the current version, that resuming an interpolation render doesn't work properly in sd-forge-deforum.
The bug seems to be in line 163 :
opts.data["CLIP_stop_at_last_layers"] = scheduled_clipskip if scheduled_clipskip is not None else opts.data["CLIP_stop_at_last_layers"]Commenting this out means the error isn't thrown, but the resume function still doesn't appear to work correctly and starts from the first frame, if the resume is attempted again.
(I'm curious if this will have any effect on the output if Clip Skip schedule is not enabled).
My own modified from sd-forge-deforum render_modes.py works with resume capabilities.
It can resume a render, accounting for missing frames, but it also does some unnecessary things, like writing out an empty text file with the exact cfg value for each image in the sequence (these seem to be very likely to have a high number of significant digits, with an unpredictably high variance between very small changes).
However, the resume loop is a little more versatile, and seems to be working, so it might be worth implementing officially.
Obviously it'd be more reasonable to find the cause of the aforementioned bug.
Not too familiar with git, so unlikely to be able to submit a fix myself, but that bug should probably be looked into first before my more 'elegant' resume solution could be considered.