Load and save a SentDebiased model

[efatmae]

Hi Nicholas,

I have another question on how to fine-tune a debiased model for the task of text classification. I checked your code in run_glue.py and wrote that following code to load and save a debiased BertforSequenceClassification model. Can you have a look at my code and let me know if I load and save the debiased model correctly or not?

Best, Fatma

` import argparse import torch from torch.nn import CrossEntropyLoss from tqdm import tqdm import os

import transformers from bias_bench.util import generate_experiment_id from bias_bench.model import models from transformers import ( AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, DataCollatorWithPadding, EvalPrediction, HfArgumentParser, PretrainedConfig, Trainer, TrainingArguments, default_data_collator, set_seed, )

if name == "main":

parser = argparse.ArgumentParser()
parser.add_argument(
    "--output_dir",
    type=str,
    default="./DebiasedSquecenceClassificationModels",
    help="Directory where results are written.",
)
parser.add_argument(
    "--debiased_model",
    action="store",
    type=str,
    default="SentenceDebiasBertForSequenceClassification",
)

parser.add_argument(
    "--model_name_or_path",
    action="store",
    type=str,
    default="bert-base-uncased",
)
parser.add_argument(
    "--bias_direction",
    action="store",
    type=str,
    default = "/home/fatma/Dropbox/Tensorbook/Bias_in_contextual_word_embeddings/IBM_cluster_jobs/Debias_LM/Sent_Debias/results/subspace_m-BertModel_c-bert-base-uncased_t-gender.pt",
    help="Path to the file containing the pre-computed bias direction for SentenceDebias.",
)
parser.add_argument(
    "--bias_type",
    action="store",
    type=str,
    choices=["gender"],
    default="gender",
    help="The type of bias to compute the bias subspace for.",
)

args = parser.parse_args()
print(f"Parameters: {args}")

# Override loaded the model.
kwargs = {}
if args.bias_direction is not None:
    # Load the pre-computed bias direction for SentenceDebias.
    bias_direction = torch.load(args.bias_direction)
    kwargs["bias_direction"] = bias_direction

print("=" * 40)
print(f"Loading: {args.model}")

tokenizer = transformers.AutoTokenizer.from_pretrained(args.model_name_or_path)
config = AutoConfig.from_pretrained(args.model_name_or_path)
inputs = tokenizer("Hello, Women are nurses", return_tensors="pt")
labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
print(labels)

model = getattr(models, args.debiased_model)(args.model_name_or_path, config=config, **kwargs)

import torch.nn.functional as F

output = model(**inputs, labels=labels)
print("debiased model output", F.softmax(output.logits, dim=-1))

print("=" * 40)
print(f"Saving: {args.model}")

experiment_id = generate_experiment_id(
    name="subspace",
    model=args.model,
    model_name_or_path=args.model_name_or_path,
    bias_type=args.bias_type,
)
#checkpoint = {'model': model, 'state_dict': model.state_dict()}
model.save_pretrained(f"{args.output_dir}/{experiment_id}")
tokenizer.save_pretrained(f"{args.output_dir}/{experiment_id}")

`

[ncmeade]

Hi @efatmae,

Yes, this looks correct! Let me know if you have any other questions about fine-tuning with debiasing.

[ncmeade]

Closing this now. Feel free to re-open if you have additional questions.