Not getting valid preds from self.regressor.forward()

[Lucky-Stark-01]

Could you please check, why I'm getting not valid results (activity predictions) after fine-tuning the meta model: actfound. This is the code I've written separately for finetuning and inferencing with validation dataset:

def finetune(args, epoch, model, dataloader, is_finetune=True):
    kname = "layer_dict.linear.weights".replace(".", "-")
    wname = "layer_dict.linear.weights"
    print("Inner loop learning rates:",model.inner_loop_optimizer.names_learning_rates_dict[kname])
    cir_num = args.test_repeat_num
    r2_list = []
    R2os_list = []
    rmse_list = []
    res_dict = {}
    print(cir_num)
    for cir in range(cir_num):
        if is_finetune:
            test_data_all = dataloader.get_test_batches(repeat_cnt=cir)

        if args.knn_maml:
            test_data_all = [x for x in test_data_all]

        for step, cur_data in enumerate(test_data_all):

            losses, per_task_target_preds, final_weights, per_task_metrics = model.run_validation_iter(cur_data)
            r2_list.append(per_task_metrics[0]["r2"])
            R2os_list.append(per_task_metrics[0]["R2os"])
            rmse_list.append(per_task_metrics[0]["rmse"])
            assay_name = cur_data[3][0]
            if assay_name not in res_dict.keys():
                res_dict[assay_name] = []
            res_dict[assay_name].append(per_task_metrics[0])

    rmse_i = np.mean(rmse_list)
    median_r2 = np.median(r2_list, 0)
    mean_r2 = np.mean(r2_list, 0)
    valid_cnt = len([x for x in r2_list if x > 0.3])
    print(
        'epoch is: {}, mean rmse is: {:.3f}'.
        format(epoch, rmse_i))
    print(
        'epoch is: {}, r2: mean is: {:.3f}, median is: {:.3f}, cnt>0.3 is: {:.3f}'.
        format(epoch, mean_r2, median_r2, valid_cnt))
    median_r2os = np.median(R2os_list, 0)
    mean_r2os = np.mean(R2os_list, 0)
    valid_cnt = len([x for x in R2os_list if x > 0.3])
    print(
        'epoch is: {}, R2os: mean is: {:.3f}, median is: {:.3f}, cnt>0.3 is: {:.3f}'.
        format(epoch, mean_r2os, median_r2os, valid_cnt))
    final_tensor = final_weights[0][wname].clone().detach().requires_grad_(True)
    print("Final Weights:",final_weights)
    model.inner_loop_optimizer.names_learning_rates_dict[kname]=final_tensor
    print("Final MOdel weights:",model.inner_loop_optimizer.names_learning_rates_dict[kname])
    # exit()
    return res_dict, model

def save_model(model,save_path):
    torch.save(model,f=save_path)    

    return save_path

def finetune_main(args,dataloader):
    model = system_selector(args)(args=args, input_shape=(2, args.dim_w))
    args.meta_batch_size = 1
    model_file = '{0}/{2}/model_{1}'.format(args.logdir, args.test_epoch, exp_string)
    if not os.path.exists(model_file):
        model_file = '{0}/{1}/model_best'.format(args.logdir, exp_string)
    try:
        model.load_state_dict(torch.load(model_file))
    except Exception as e:
        print(e)
        model.load_state_dict(torch.load(model_file), strict=False)

    if args.knn_maml:
        old_sup_num = args.test_sup_num
        args.test_sup_num = 0.90
        args.knn_maml = False
        prepare_assay_feat(args, model, dataloader)
        args.test_sup_num = old_sup_num
        args.knn_maml = True
        model.prepare_knn_maml(dataloader)

    res_dict, model = finetune(args, args.test_epoch, model, dataloader)

    write_dir = os.path.join(args.test_write_file, args.model_name)
    if not os.path.exists(write_dir):
        os.system(f"mkdir -p {write_dir}")
    out_path = f"{write_dir}/without_act_num_{args.test_sup_num}.json"
    print("write result to", out_path)
    print("\n\n\n\n")
    json.dump(res_dict, open(out_path, "w"))

    print("////////////Saving Finetuned Model :")
    save_path = args.save_finetuned_model
    print("Saving Model Done!!!!!!")
    save_model(model,save_path)

    return save_path

def inference_main(args):
    # model = system_selector(args)(args=args, input_shape=(2, args.dim_w))
    dataloader = dataset_constructor(args)

    args.meta_batch_size = 1
    val_data_all = dataloader.get_val_batches()

    model_file = finetune_main(args,dataloader)

    try:
        model = torch.load(model_file)
    except Exception as e:
        print(e)

    save_dir = "/home/ubuntu/ActFound/results"
    os.makedirs(save_dir,exist_ok=True)
    assay_preds = {}
    for cur_data in tqdm(val_data_all):
        x_task = cur_data[0][0]
        x_task = x_task.float().cuda()
        assay_name = cur_data[3][0].replace("/", "_")
        embs, preds = model.regressor.forward(x_task,num_step=0)
        print("preds:",preds)
        preds = preds.detach().cpu().numpy().tolist()
        assay_preds[assay_name]=preds

    save_output = os.path.join(save_dir,"preds.json")
    json.dump(assay_preds,open(save_output,'w'))
    print("results saved to:",save_output)
    print("Done!!!!!!!!!!")

Output Predictions I got: [-0.6336, -0.4449, -0.5665, -0.6252, -0.4101, -0.9025, -0.9602, -0.7823, -0.8399, -0.7683, -0.8970, -0.7017, -0.7930, -0.7291, -0.9378, -0.2267, -0.4170, 0.0460, -0.1171, -0.4783, -0.5798, -1.5286, -0.2624, -0.3334, -0.3932, -1.0339, -0.9790, -1.0076, -0.2937, -1.1409, -1.1365]

[BFeng14]

Thank you for your interest on Actfound, Actfound is a meta learning model designed for few-shot bioactivity prediction, and we only finetune the last linear layer during finetuning process, which is inplemented in model.run_validation_iter(cur_data). The finetuning process is implemented in the inner_loop function of system_base:

    def inner_loop(self, x_task, y_task, assay_idx, split, is_training_phase, epoch, num_steps):
        task_losses = []
        per_step_loss_importance_vectors = self.get_per_step_loss_importance_vector()
        support_loss_each_step = []
        sup_num = torch.sum(split)
        names_weights_copy = self.get_inner_loop_parameter_dict(self.regressor.named_parameters())
        use_second_order = self.args.second_order and epoch > self.args.first_order_to_second_order_epoch
        for num_step in range(num_steps):
            support_loss, support_preds = self.net_forward(x=x_task,
                                                           y=y_task,
                                                           assay_idx=assay_idx,
                                                           split=split,
                                                           is_support=True,
                                                           weights=names_weights_copy,
                                                           backup_running_statistics=
                                                           True if (num_step == 0) else False,
                                                           training=True, num_step=num_step)
            support_loss_each_step.append(support_loss.detach().cpu().item())
            if support_loss >= support_loss_each_step[0] * 10 or sup_num <= 5:
                pass
            else:
                names_weights_copy = self.apply_inner_loop_update(loss=support_loss,
                                                                  names_weights_copy=names_weights_copy,
                                                                  use_second_order=use_second_order if is_training_phase else False,
                                                                  current_step_idx=num_step,
                                                                  sup_number=torch.sum(split))

            if is_training_phase:
                is_multi_step_optimize = self.args.use_multi_step_loss_optimization and epoch < self.args.multi_step_loss_num_epochs
                is_last_step = num_step == (self.args.num_updates - 1)
                if is_multi_step_optimize or is_last_step:
                    target_loss, target_preds = self.net_forward(x=x_task,
                                                                 y=y_task,
                                                                 assay_idx=assay_idx,
                                                                 split=split,
                                                                 weights=names_weights_copy,
                                                                 backup_running_statistics=False,
                                                                 training=True,
                                                                 num_step=num_step)
                    if is_multi_step_optimize:
                        task_losses.append(per_step_loss_importance_vectors[num_step] * target_loss)
                    elif is_last_step:
                        task_losses.append(target_loss)

        return names_weights_copy, support_loss_each_step, task_losses

The parameter of last linear layer after finetuning is given in "final_weights", and the evaluation result after finetuning is given in "per_task_target_preds". So you only need to run finetune and save the result you want, without need to run inference_main(). Please let me know if there are any other questions~~~

[Lucky-Stark-01]

Here, In this code i'll explain briefly what I did for predicting the activity value of unknown compounds. I considered only two types of assay namely "test" and "valid". I've created a data-loader for that data with split_train_test_val from json file. While looking into this problem simply, it's a task of fine-tuning and getting predictions for the unknown compounds. Point-1: You're saying that to use model.run_validation_iter(cur_data) to finetune and save predictions per_task_target_preds. But my intuition here is to finetune the base meta model with my assay type data and get activity predictions for similar unknown compounds (This is my core problem). Point-2: In this code model.run_validation_iter(cur_data), you're using actual y lables and getting predictions relative to the actual known values.

• Description: After getting these values from `inner_loop` function, `names_weights_copy, support_loss_each_step, task_losses`, you're sending them to net_forward() function with updated weights. After that you're getting some out_value predictions from self.regressor.forward() method. But those values doesn't make any sense. You're then using those values and doing some calculations with actual y_task values and sending us the predictions. So, why you're taking actual labels along with out_values? What is the exact meaning of those predictions from self.net_forwards function?

• Here, I updated the base model linear layer weights with the final weights I got after the finetuning the model and doing model.regressor.forward() on unknown x_task data to get some values (y predictions ~ activity values)...But I'm not getting valid predictions. How to proceed further and what is the correct code to get valid activity predictions for unknown compounds after finetuning the base model with specific assay.

[BFeng14]

Hi, I think your question is to predict the activity of compounds with unknown activity from a specific assay(your test or valid assay), providing that there are some compounds with known activity in this assay (Please correct me if I was wrong). This code was implemented to do a large number of comparative experiments in several datasets, and what you want is not implemented here. I will implement it in the next few days to make Actfound more convenient to use.

As for the self.regressor.forward() method, x means input feature of all compounds(2048-dim), y means their activity, split means the split of compounds for this assay [1. means the compounds used for finetuning (which we usually call "support set" in meta-learning), and 0. means the compounds used for testing (which we usually call "target set" in meta-learning) ]. The Actfound base model is finetuned on the "support set", and give prediction on the "target set". Actfound is a pairwise learning model, which means the model predict the activity difference between compounds in an assay, and then reconstruct the final predicted activity of compounds in "target set", so it is ensential to input y for the compounds in "support set", and the y of the compounds in the "target set" is not used, please don't worry about data leakage problem here.

The data partitioning part of meta-learning is different from that of general machine learning tasks, which might be a little bit confusing. Please don't worry about that cause I will provide you a convident method very soon. Please let me know if there are any more questions.