yunjinchoidev
commented
1 year ago 어느정도 CoRE 깃헙에 나와 있는 core.py 파일을 수정해서 커스텀하려고 했으나 몇 가지 문제점이 있습니다.
-
체크포인트를 어떻게 이식시킬지 ?
TypeError: expected str, bytes or os.PathLike object, not NoneType에러가 나와서 진척이 없다. -
luke_prob_mask_2 이란 무엇인가? 논문 상 라벨 편향성을 제거해주는 부분으로 보인다. 그런데 어떤 기준에서 값을 설정해주는 지 모르겠음.. core.py 파일에서는 외부에서 값을 불러오는 것 으로 보임. 값을 살펴보면 전부 다 (1. 0. 0. 0. 0. .... ) 으로 되어 있는 듯 함.
-
label_constraint 란 무엇인가 ? 이 부분에 대해 논문에서 상세한 설명은 없는 것 처럼 보이는데. 단순히 상수로 보정값으로 보임. 그런데 core.py 파일에서
label_constraint를 직접 불러와서 읽어보면 굉장히 다채롭게 값이 부여가 되어 있음. 어떤 편향을 제거해주는 값으로 보이는데. 추측상 eda를 진행해서 어떤 편향을 라벨별로 제거해줘야 하지 않나 싶다. -
core.py 에서는 일반 data 에 대해서 라벨이 주어진 상태에서 진행하여 CoRE 방법을 진행한다. 그 상태에서 keys(라벨값) 들을 추출 하는 것을 보임. 하지만 본인이 진행하려고 하는 방식은 test data 에 대해서 단순히 예측을 하려고 하는 것인데. 그러면 keys 값들을 모델의 예측값을 쓰면 되는 것인가?
import joblib
import numpy as np
import scipy.special as sp
from transformers import LukeTokenizer, LukeForEntityPairClassification
import json
import torch
from tqdm import trange
from sklearn.metrics import f1_score
import pandas as pd
import os
import csv
import torch.nn.functional as F
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import BertTokenizer, RobertaForSequenceClassification
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import LukeTokenizer, LukeForSequenceClassification
# getF1 function calculates the f1 score for relation extraction.
# def getF1(key, prediction):
# min_label, max_label = min(key), max(key)
# f1_ls = []
# recall_ls = []
# prec_ls = []
# for label_id_ in range(min_label, max_label + 1):
# correct_by_relation = ((key == prediction) & (prediction == label_id_)).astype(np.int32).sum()
# guessed_by_relation = (prediction == label_id_).astype(np.int32).sum()
# gold_by_relation = (key == label_id_).astype(np.int32).sum()
# prec_micro = 1.0
# if guessed_by_relation > 0:
# prec_micro = float(correct_by_relation) / float(guessed_by_relation)
# recall_micro = 1.0
# if gold_by_relation > 0:
# recall_micro = float(correct_by_relation) / float(gold_by_relation)
# f1_micro = 0.0
# if prec_micro + recall_micro > 0.0:
# f1_micro = 2.0 * prec_micro * recall_micro / (prec_micro + recall_micro)
# f1_ls.append(f1_micro)
# recall_ls.append(recall_micro)
# prec_ls.append(prec_micro)
# return sum(f1_ls[1 :]) / len(f1_ls[1 :])
def getF1(key, prediction):
return f1_score(key, prediction, average='micro')
# keys is the numpy array storing the sentence-wise relation labels.
# label_constraint is the numpy array storing the relation constraints based on the entity types.
# ID_TO_LABEL is the dictionary projects the label id in keys to the textual entity relations.
# These variables are stored after the pre-processing.
# keys, label_constraint, ID_TO_LABEL, luke_prob_mask_2 = joblib.load('preprocess.data')
def convert_csv_to_list(file_path):
data_list = []
with open(file_path, 'r', encoding='utf-8') as file:
reader = csv.DictReader(file)
for row in reader:
text = row['sentence']
subj_entity = eval(row['subject_entity'])
obj_entity = eval(row['object_entity'])
label = row['label']
entity_spans = [(subj_entity['start_idx'], subj_entity['end_idx']),
(obj_entity['start_idx'], obj_entity['end_idx'])]
entity_type = (subj_entity['type'], obj_entity['type'])
# 텍스트 길이가 512보다 길 경우 자르기
# if len(text) > 500:
# text = text[:500]
data = {
'text': text,
'entity_spans': entity_spans,
'label': label,
'entity_type': entity_type
}
data_list.append(data)
return data_list
test_examples = convert_csv_to_list('test_data.csv')
with open('test_example.csv', 'w', encoding='utf-8') as f:
# json pretty store like
# {
# 'text': 'text',
# 'entity_spans': [(0, 1), (2, 3)],
# 'label': 'label',
# 'entity_type': ('type1', 'type2')
# }
# pretty store
for example in test_examples:
f.write(json.dumps(example, ensure_ascii=False))
f.write('\n')
keys, label_constraint, ID_TO_LABEL, luke_prob_mask_2 = None, None, None, None
ID_TO_LABEL = {0: 'no_relation', 1: 'org:top_members/employees', 2: 'org:members', 3: 'org:product', 4: 'per:title', 5: 'org:alternate_names', 6: 'per:employee_of', 7: 'org:place_of_headquarters', 8: 'per:product', 9: 'org:number_of_employees/members', 10: 'per:children', 11: 'per:place_of_residence', 12: 'per:alternate_names', 13: 'per:other_family', 14: 'per:colleagues', 15: 'per:origin', 16: 'per:siblings', 17: 'per:spouse', 18: 'org:founded', 19: 'org:political/religious_affiliation', 20: 'org:member_of', 21: 'per:parents', 22: 'org:dissolved', 23: 'per:schools_attended', 24: 'per:date_of_death', 25: 'per:date_of_birth', 26: 'per:place_of_birth', 27: 'per:place_of_death', 28: 'org:founded_by', 29: 'per:religion'}
LABEL_TO_ID = {'no_relation': 0, 'org:top_members/employees': 1, 'org:members': 2, 'org:product': 3, 'per:title': 4, 'org:alternate_names': 5, 'per:employee_of': 6, 'org:place_of_headquarters': 7, 'per:product': 8, 'org:number_of_employees/members': 9, 'per:children': 10, 'per:place_of_residence': 11, 'per:alternate_names': 12, 'per:other_family': 13, 'per:colleagues': 14, 'per:origin': 15, 'per:siblings': 16, 'per:spouse': 17, 'org:founded': 18, 'org:political/religious_affiliation': 19, 'org:member_of': 20, 'per:parents': 21, 'org:dissolved': 22, 'per:schools_attended': 23, 'per:date_of_death': 24, 'per:date_of_birth': 25, 'per:place_of_birth': 26, 'per:place_of_death': 27, 'org:founded_by': 28, 'per:religion': 29}
label_constraint = np.zeros((len(test_examples), len(ID_TO_LABEL.keys())))
luke_prob_mask_2 = np.ones((len(test_examples), len(ID_TO_LABEL.keys())))
# # # Load the model checkpoint
# model = LukeForEntityPairClassification.from_pretrained("studio-ousia/luke-large-finetuned-tacred")
model = LukeForSequenceClassification.from_pretrained("studio-ousia/mluke-large")
checkpoint_path = '../level2_klue-nlp-07/baseline/ckpt/studio-ousia/mluke-large.ckpt'
checkpoint = torch.load(checkpoint_path)
# model.load_state_dict(torch.load(checkpoint_path))
# checkpoint_path = '../level2_klue-nlp-07/baseline/ckpt/studio-ousia/mluke-large.ckpt'
# model = RobertaForSequenceClassification.from_pretrained('studio-ousia/mluke-large')
# model = RobertaForSequenceClassification.from_pretrained(model)
model.eval()
model.to("cuda")
# # Load the tokenizer
tokenizer = LukeTokenizer.from_pretrained("studio-ousia/mluke-large")
# tokenizer = RobertaTokenizer.from_pretrained('klue/roberta-large')
# Remove unwanted keys from the checkpoint
compatible_checkpoint = {k: v for k, v in checkpoint.items() if k in model.state_dict()}
# Load the compatible keys into the model
model.load_state_dict(compatible_checkpoint)
batch_size = 128
# # # produce the original testing result on the test set of TACRED
pred_ls = []
label_ls = []
model.config.id2label = ID_TO_LABEL
max_length = 512 # 모델의 최대 시퀀스 길이
keys = []
for batch_start_idx in trange(0, len(test_examples), batch_size):
batch_examples = test_examples[batch_start_idx:batch_start_idx + batch_size]
texts = [example["text"] for example in batch_examples]
entity_spans = [example["entity_spans"] for example in batch_examples]
gold_labels = [example["label"] for example in batch_examples]
# inputs = tokenizer(texts, entity_spans=entity_spans, return_tensors="pt", padding=True)
# inputs = tokenizer(texts, entity_spans=entity_spans, padding="max_length", truncation=True, max_length=max_length, return_tensors="pt")
inputs = tokenizer(texts, padding="max_length", truncation=True, max_length=512, return_tensors="pt")
inputs["entity_spans"] = entity_spans
inputs = inputs.to("cuda")
inputs = inputs.to("cuda")
# print(inputs)
with torch.no_grad():
outputs = model(**inputs)
predicted_indices = outputs.logits.argmax(-1)
predicted_labels = [model.config.id2label[index.item()] for index in predicted_indices]
pred_ls.append(outputs.logits.cpu().numpy())
label_ls = label_ls + [label_ for label_ in gold_labels]
keys = np.concatenate(pred_ls, axis=0).argmax(axis=1)
luke_prob, luke_id_to_label = np.concatenate(pred_ls, axis = 0), model.config.id2label
# print("luke_prob", luke_prob)
# print("luke_id_to_label", luke_id_to_label)
# produce the counterfactual predictions to distill the entity bias
pred_ls = []
label_ls = []
for batch_start_idx in trange(0, len(test_examples), batch_size):
batch_examples = test_examples[batch_start_idx:batch_start_idx + batch_size]
texts = [example["text"] for example in batch_examples]
entity_spans = [example["entity_spans"] for example in batch_examples]
gold_labels = [example["label"] for example in batch_examples]
texts = [
texts[i_][entity_spans[i_][0][0] : entity_spans[i_][0][1]]
+
' '
+
texts[i_][entity_spans[i_][1][0] : entity_spans[i_][1][1]]
for i_ in range(len(texts))
]
entity_spans = [
[(0, entity_spans[i_][0][1] - entity_spans[i_][0][0]),
(entity_spans[i_][0][1] - entity_spans[i_][0][0] + 1,
entity_spans[i_][0][1] - entity_spans[i_][0][0] + 1 + entity_spans[i_][1][1] - entity_spans[i_][1][0]
)]
for i_ in range(len(texts))
]
# inputs = tokenizer(texts, entity_spans=entity_spans, return_tensors="pt", padding=True)
# inputs = tokenizer(texts, entity_spans=entity_spans, padding="max_length", truncation=True, max_length=max_length, return_tensors="pt")
inputs = tokenizer(texts, padding="max_length", truncation=True, max_length=512, return_tensors="pt")
inputs["entity_spans"] = entity_spans
inputs = inputs.to("cuda")
inputs = inputs.to("cuda")
with torch.no_grad():
outputs = model(**inputs)
predicted_indices = outputs.logits.argmax(-1)
predicted_labels = [model.config.id2label[index.item()] for index in predicted_indices]
pred_ls.append(outputs.logits.cpu().numpy())
label_ls = label_ls + [label_ for label_ in gold_labels]
luke_prob_mask = np.concatenate(pred_ls, axis = 0)
# # normalize the predicted logits as probabilities by softmax
luke_prob = sp.softmax(luke_prob, axis = 1) # 기존꺼
luke_prob_mask = sp.softmax(luke_prob_mask, axis = 1) # 엔티티 정보만 고려 한 것.
# print("luke_prob", luke_prob)
# print("luke_prob_mask", luke_prob_mask)
# transform the luke prediction indices to the original label indices.
luke_label_to_id = {value_ : key_ for key_, value_ in luke_id_to_label.items()}
org_to_luke = [luke_label_to_id[ID_TO_LABEL[i_]] for i_ in range(len(luke_label_to_id.values()))]
luke_prob = luke_prob[:, org_to_luke]
luke_prob_mask_1 = luke_prob_mask[:, org_to_luke]
luke_preds = luke_prob.argmax(1)
luke_preds_tde = luke_prob_mask_1.argmax(1)
# print("luke_preds", luke_preds)
# print("luke_preds_tde", luke_preds_tde)
# print("luke_prob_size", luke_prob.shape)
# print("luke_prob_mask_size", luke_prob_mask_1.shape)
# submission 작성
submission = pd.read_csv('./sample_submission.csv')
submission['pred_label'] = luke_preds.tolist()
submission['probs'] = luke_prob.tolist()
submission.to_csv('./submission.csv', index=False)
with open('./go1.csv', 'w') as f:
f.write(submission.to_csv(index=False))
# filter the challenge set that the relation labels implied by the entity bias does not exist in the sentence.
challenge_set = [i_ for i_ in range(len(luke_prob)) if luke_preds_tde[i_] != keys[i_]]
with open('./challenge_set.txt', 'w') as f:
f.write(str(challenge_set))
print("challenge_set", challenge_set)
# challenge_set을 NumPy 배열로 변환합니다.
challenge_set = np.array(challenge_set)
# challenge_set을 사용하여 keys 리스트를 인덱싱합니다.
keys = np.array(keys)[challenge_set]
# keys = keys[challenge_set]
luke_preds = luke_preds[challenge_set]
luke_prob = luke_prob[challenge_set]
print("~~~~~~~~~~~~~~~~~``")
print("luke_preds", luke_preds.shape)
print("luke_prob", luke_prob.shape)
print("~~~~~~~~~~~~~~~~~``")
luke_prob_mask_1 = luke_prob_mask_1[challenge_set]
luke_prob_mask_2 = luke_prob_mask_2[challenge_set]
label_constraint = label_constraint[challenge_set]
# print(keys)
# print(luke_preds)
print('f1 score before bias mitigation: ', getF1(keys, luke_preds))
lamb_1 = -1.6
lamb_2 = 0.1
new_preds = (luke_prob + lamb_1 * luke_prob_mask_1 + lamb_2 * luke_prob_mask_2 + label_constraint).argmax(1)
print('f1 score after bias mitigation: ', getF1(keys, new_preds))
# new_preds의 index 에 맞는 것만 submission.csv 파일에 쓰기 위해 저장합니다.
# 데이터프레임의 복사본을 생성합니다.
submission = submission.copy()
# 데이터프레임의 행을 순회합니다.
for i, row in submission.iterrows():
print("origin", i, row, row['pred_label'], row['probs'])
# 인덱스 i가 challenge_set 리스트에 있는지 확인합니다.
if i in challenge_set.tolist():
# 'pred_label' 열을 업데이트합니다.
submission.at[i, 'pred_label'] = new_preds.tolist()[challenge_set.tolist().index(i)]
# 'probs' 열을 업데이트합니다.
submission.at[i, 'probs'] = luke_prob.tolist()[challenge_set.tolist().index(i)]
print("update", submission.at[i, 'pred_label'], submission.at[i, 'probs'])
else:
print("Index not found in challenge_set:", i)
with open('./go2.csv', 'w') as f:
f.write(submission.to_csv(index=False))
print(new_preds)
print(new_preds.shape)
all_preds = []
all_probs = []
submission = pd.read_csv('./submission.csv')
# 모든 예측을 수집합니다.
for i in range(len(luke_prob)):
if i in challenge_set:
pred = new_preds[challenge_set.tolist().index(i)]
prob = luke_prob[challenge_set.tolist().index(i)]
else:
pred = luke_preds[i]
prob = luke_prob[i]
all_preds.append(ID_TO_LABEL[pred])
all_probs.append(F.softmax(torch.tensor(prob), dim=-1).tolist())
for i in range(len(luke_prob)):
if i in challenge_set:
pred = new_preds[challenge_set.tolist().index(i)]
prob = luke_prob[challenge_set.tolist().index(i)]
else:
pred = luke_preds[i]
prob = luke_prob[i]
all_preds.append(ID_TO_LABEL[pred])
all_probs.append(F.softmax(torch.tensor(prob), dim=-1).tolist())
# submission DataFrame을 만듭니다.
submission_data = []
for i, (pred, prob) in enumerate(zip(all_preds, all_probs)):
submission_dict = {"id": i, "pred_label": pred, "probs": json.dumps(prob)}
submission_data.append(submission_dict)
submission_df = pd.DataFrame(submission_data)
# submission 파일을 CSV 형식으로 저장합니다.
submission_df.to_csv('submission.csv', index=False)
JLake310
Background
Should We Rely on Entity Mentions for Relation Extraction? Debiasing Relation Extraction with Counterfactual Analysis, NAACL2022논문에서 제시한 CoRE 기법이 RE Task 에서 높은 성능을 보였기 때문에 구현을 시도한다.To Do
See Also
21