Issue while using multiprocessing

[sumbalakram]

While using multiprocessing for my task my program gets the exception after some generations. Sometimes it works fine for all generations and stops working abruptly and generates "NoneType object is unsubscriptable" or Pool halt errors. I go through the documentation but there is no detail provided about how multiprocessing is working and what to do and what not to do while using this. Please guide me in this regard to what can be the possible issue. My code is

def fitness_func(solution, solution_idx):

cal ISP for each gamma in population => ehanced image

conf = Config_file_handling.YAMLfunctions() config = conf.read_yaml('Default_isp_conf.yml') config['gac']['gamma'] = float(solution[0]) config['nlm']['h'] = int(solution[1]) conf.write_yaml('Default_isp_conf.yml', config) config = conf.read_yaml('config.yml') IQM = ISP(config) iqm = mean(IQM) return round(iqm['PSNR'],4)

Create the GA instance

ga_instance = pygad.GA(num_generations=10, num_parents_mating=2, sol_per_pop=20, num_genes=2, gene_type=gene_type, gene_space=gene_space, fitness_func=fitness_func, on_generation=on_generation, parent_selection_type="rank", keep_parents=1, crossover_type='single_point', crossover_probability=0.8, mutation_type='random', mutation_probability=0.2, allow_duplicate_genes=False,

stop_criteria="saturate_5",

                   save_best_solutions=True,
                   save_solutions=True,
                   parallel_processing=['process', 5]
                   )

t1 = time.time()

Run the GA

ga_instance.run() t2 = time.time() print("Time is", t2-t1)

[ahmedfgad]

Can you share a full code to test on my end?

Some code is missing from the posted example.

import pygad
import time

def fitness_func(ga_instance, solution, solution_idx):
    #cal ISP for each gamma in population => ehanced image
    conf = Config_file_handling.YAMLfunctions()
    config = conf.read_yaml('Default_isp_conf.yml')
    config['gac']['gamma'] = float(solution[0])
    config['nlm']['h'] = int(solution[1])
    conf.write_yaml('Default_isp_conf.yml', config)
    config = conf.read_yaml('config.yml')
    IQM = ISP(config)
    iqm = mean(IQM)
    return round(iqm['PSNR'],4)

ga_instance = pygad.GA(num_generations=10,
                       num_parents_mating=2,
                       sol_per_pop=20,
                       num_genes=2,
                       gene_type=gene_type,
                       gene_space=gene_space,
                       fitness_func=fitness_func,
                       on_generation=on_generation,
                       parent_selection_type="rank",
                       keep_parents=1,
                       crossover_type='single_point',
                       crossover_probability=0.8,
                       mutation_type='random',
                       mutation_probability=0.2,
                       allow_duplicate_genes=False,
                       #stop_criteria="saturate_5",
                       save_best_solutions=True,
                       save_solutions=True,
                       parallel_processing=['process', 5])

t1 = time.time()
ga_instance.run()
t2 = time.time()
print("Time is", t2-t1)

[sumbalakram]

import pygad
import time

def mean(IQM):
    mean_dict = {}
    for key in IQM[0].keys():
          mean_dict[key] = sum(d[key] for d in IQM) / len(IQM)
    return mean_dict

def fitness_func(ga_instance, solution, solution_idx):
    #cal ISP for each gamma in population => ehanced image
    conf = Config_file_handling.YAMLfunctions()
    config = conf.read_yaml('Default_isp_conf.yml')
    config['gac']['gamma'] = float(solution[0])
    config['nlm']['h'] = int(solution[1])
    conf.write_yaml('Default_isp_conf.yml', config)
    config = conf.read_yaml('config.yml')
    IQM = ISP(config)
    iqm = mean(IQM)
    return round(iqm['PSNR'],4)

ga_instance = pygad.GA(num_generations=10,
                       num_parents_mating=2,
                       sol_per_pop=20,
                       num_genes=2,
                       gene_type=gene_type,
                       gene_space=gene_space,
                       fitness_func=fitness_func,
                       on_generation=on_generation,
                       parent_selection_type="rank",
                       keep_parents=1,
                       crossover_type='single_point',
                       crossover_probability=0.8,
                       mutation_type='random',
                       mutation_probability=0.2,
                       allow_duplicate_genes=False,
                       #stop_criteria="saturate_5",
                       save_best_solutions=True,
                       save_solutions=True,
                       parallel_processing=['process', 5])

t1 = time.time()
ga_instance.run()
t2 = time.time()
print("Time is", t2-t1)

Main:

`def image_quality_matrices(original_img, modified_img) -> dict: """This functions handles the IQM in main"""

iqm = IQM.ImageQualityMatrices(original_img)
iqm_dict = {'PSNR': iqm.PSNR(modified_img), 'SSIM': iqm.SSIM(modified_img), 'UQI': iqm.uqi(P=modified_img)}
#print(iqm_dict)

return iqm_dict

def ISP(conf): """ Executes the Fast Open ISP in main """ list_iqms = []

Create Log File directory if not available

os.makedirs(conf['log_file_directory'], exist_ok=True)

# Checks the input Directory
if not os.path.isdir(conf['images_input_directory']):
    print("Please specify the correct input path")
    sys.exit()
else:
    for image in glob.glob(conf['images_input_directory'] + '*'):
        log_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S,')
        isp_conf = Config_file_handling.YAMLfunctions()
        isp_params = isp_conf.read_yaml(conf['configuration_file_path'])
        file_name = image.split('\\')[-1].split('.')[0]
        print(file_name)
        try:

            # Parameters Optimization
            # optimize = Parameters_optimization.Optimization(image)
            # optimize.param_optimize(conf['configuration_file_path'])

            # Getting the image size from metadata
            Image_metadata.metadata_conf(image, conf['configuration_file_path'])

            # Running the ISP
            isp = isp_run.RunISPs(image_file=image, output_dir=conf['images_output_directory'], add_fileTime=conf[
                'add_time_along_with_FileName'])
            ISPOut_dict = isp.runISP(conf['configuration_file_path'], isp_name=conf['available_ISPs'][conf[
                "ISP_to_use"]-1])

            if conf['Perform IQM Checks']['status']:
                # Getting the Path of Ground Truth Image
                file_name = image.split('\\')[-1]
                ext = file_name.split('.')[-1]
                o_image_name = file_name.replace(ext, 'jpg')
                o_image_path = f'{conf["Perform IQM Checks"]["ground_truth_directory"]}\\{o_image_name}'

                # Performing Image Quality Checks
                iqms = image_quality_matrices(o_image_path, ISPOut_dict['OutputImage'])
                #up_iqms = {"Image": file_name}
                #up_iqms.update(iqms)
                list_iqms.append(iqms)
                print(list_iqms)

                # Writing in log file
                with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file:
                    log_file.write(f'{log_time} INFO, {ISPOut_dict["FileName"]}, {iqms}, {isp_params}\n')
            else:
                with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file:
                    log_file.write(f'{log_time} INFO, {ISPOut_dict["FileName"]}, {isp_params}\n')

        except Exception as error:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fName = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            template = "{0}, File {1}, Line {2}\n{3}"
            message = template.format(exc_type, fName, exc_tb.tb_lineno, error.args).replace('\n', ' ')
            with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file:
                log_file.write(f'{log_time} {file_name}, {message}, {isp_params}\n') 

return list_iqms`

[ahmedfgad]

Thanks for sharing the code.

Please provide the following to run the code.

• Config_file_handling
• isp_run
• Image_metadata
• gene_type
• gene_space
• on_generation