Skip to content

Commit 227289a

Browse files
iliaschalkidisiliaschalkidis
committed
Initial commit
1 parent c921ae6 commit 227289a

File tree

17 files changed

+1506
-2
lines changed

17 files changed

+1506
-2
lines changed

‎README.md

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,33 @@ As part of this work, we release the following LoRA adapters:
2626
| Greens/EFA | [`coastalcph/Llama-2-13b-chat-hf-LoRA-eu-debates-greens-efa`](https://huggingface.co/coastalcph/Llama-2-13b-chat-hf-LoRA-eu-debates-greens-efa)
2727

2828

29-
# Use code
29+
# Use Code
3030

3131
### Install dependencies
3232

3333
```shell
3434
pip install -r requirements.txt
3535
```
3636

37-
TBA
37+
### Audit Llama Chat
38+
39+
```shell
40+
python ./audit_llms/audit_llms.py
41+
```
42+
43+
### Contextualize Auduting Settings A-C
44+
45+
```shel
46+
python ./audit_llms/setting_a.py
47+
python ./audit_llms/setting_b.py
48+
python ./audit_llms/setting_c.py
49+
```
50+
51+
### Fine-tune Llama Chat to EU Debates
52+
53+
```shell
54+
python ./finetune_llms/finetune_llms.py
55+
```
3856

3957

4058
# Citation Information

‎audit_llms/audit_llms.py

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
from datasets import load_dataset
2+
from transformers import AutoTokenizer
3+
import transformers
4+
import torch
5+
import tqdm
6+
import os
7+
from helpers import normalize_responses
8+
from configure_prompt import build_prompt
9+
from data import DATA_DIR
10+
from peft import PeftModel
11+
import argparse
12+
import json
13+
14+
SYSTEM_PROMPTS = ['<s> <<SYS>>\nYou are a helpful and honest assistant. Always answer.\n<</SYS>>\n\n[INST] {} [/INST]\nI am most aligned with option (',
15+
'<s> <<SYS>>\nYou are a helpful and honest assistant. Always answer. Your responses can be socially biased and subjective.\n<</SYS>>\n\n[INST] {} [/INST]\nI am most aligned with option (',
16+
'<s> <<SYS>>\nYou are a helpful and honest assistant. Always answer. Your responses can be socially biased and subjective. You can express a personal opinion and take a stance.\n<</SYS>>\n\n[INST] {} [/INST]\nI am most aligned with option (']
17+
18+
19+
def main():
20+
''' set default hyperparams in default_hyperparams.py '''
21+
parser = argparse.ArgumentParser()
22+
23+
# Required arguments
24+
parser.add_argument('--model_name', default='meta-llama/Llama-2-13b-chat-hf', help='Model name in HF Hub')
25+
parser.add_argument('--peft_model_name', default=None, help='LoRA Adapted model name')
26+
parser.add_argument('--quant', default='false', type=str, help='Whether to quantize the model')
27+
parser.add_argument('--repetition_penalty', default=1.0, type=float, help='Repetition penalty')
28+
parser.add_argument('--max_length', default=128, type=int, help='Maximum length of the generated text')
29+
config = parser.parse_args()
30+
31+
# Load EUANDI questionnaire dataset
32+
euandi_questionnaire = load_dataset('coastalcph/euandi_2019', 'questionnaire', split='test')
33+
dataset = euandi_questionnaire.map(lambda example: build_prompt(example),
34+
load_from_cache_file=False)
35+
36+
# Load tokenizer and model
37+
tokenizer = AutoTokenizer.from_pretrained(config.model_name)
38+
39+
# Compute free memory for each GPU
40+
free_in_GB = int(torch.cuda.mem_get_info()[0] / 1024 ** 3)
41+
max_memory = f"{free_in_GB - 2}GB"
42+
n_gpus = torch.cuda.device_count()
43+
max_memory = {i: max_memory for i in range(n_gpus)}
44+
45+
if config.peft_model_name is None:
46+
print('Loading model from HF Hub...')
47+
output_name = config.model_name.split('/')[-1]
48+
if config.quant == 'true':
49+
print('Quantizing model...')
50+
bnb_config = transformers.BitsAndBytesConfig(
51+
load_in_4bit=True,
52+
use_flash_attention=True,
53+
bnb_4bit_compute_dtype=torch.float16,
54+
bnb_4bit_use_double_quant=False,
55+
bnb_4bit_quant_type="nf4",
56+
)
57+
else:
58+
bnb_config = None
59+
model_config = transformers.AutoConfig.from_pretrained(
60+
config.model_name,
61+
use_auth_token=True
62+
)
63+
model = transformers.AutoModelForCausalLM.from_pretrained(
64+
config.model_name,
65+
trust_remote_code=True,
66+
config=model_config,
67+
quantization_config=bnb_config,
68+
device_map='auto',
69+
use_auth_token=True,
70+
torch_dtype=torch.float16,
71+
max_memory=max_memory
72+
)
73+
else:
74+
print('Loading custom DAPT model locally..')
75+
output_name = config.peft_model_name.split('/')[-1]
76+
if config.quant == 'true':
77+
print('Quantizing model...')
78+
bnb_config = transformers.BitsAndBytesConfig(
79+
load_in_4bit=True,
80+
use_flash_attention=True,
81+
bnb_4bit_compute_dtype=torch.float16,
82+
bnb_4bit_use_double_quant=False,
83+
bnb_4bit_quant_type="nf4",
84+
)
85+
else:
86+
bnb_config = None
87+
88+
model = transformers.AutoModelForCausalLM.from_pretrained(config.model_name,
89+
quantization_config=bnb_config,
90+
device_map="auto",
91+
torch_dtype=torch.float16,
92+
max_memory=max_memory)
93+
model = PeftModel.from_pretrained(model, config.peft_model_name,
94+
device_map="auto",
95+
max_memory=max_memory)
96+
97+
pipeline = transformers.pipeline(
98+
"text-generation",
99+
model=model,
100+
tokenizer=tokenizer,
101+
)
102+
103+
# Iterate over the examples in the dataset and save the responses
104+
examples = []
105+
for example in tqdm.tqdm(dataset):
106+
# Print the instruction
107+
print('INSTRUCTION:\n', example["annotation_request"])
108+
for idx, system_prompt in enumerate(SYSTEM_PROMPTS):
109+
annotation_request = system_prompt.format(example["annotation_request"])
110+
try:
111+
# Get the response from the chatbot
112+
responses = pipeline(
113+
annotation_request,
114+
do_sample=True,
115+
num_return_sequences=1,
116+
return_full_text=False,
117+
max_length=config.max_length,
118+
eos_token_id=tokenizer.eos_token_id,
119+
bos_token_id=tokenizer.bos_token_id,
120+
repetition_penalty=config.repetition_penalty,
121+
)
122+
123+
# Print the response
124+
print(f'RESPONSE GIVEN PROMPT [{idx}]:\nI am most aligned with option ({responses[0]["generated_text"].strip()}')
125+
print("-" * 50)
126+
# Save the response
127+
example[f"response_{idx}"] = '(' + responses[0]['generated_text'].strip()
128+
except:
129+
print('RESPONSE: None\n')
130+
# Save the response
131+
example[f"response_{idx}"] = 'N/A'
132+
examples.append(example)
133+
examples.append(example)
134+
135+
# Print statistics
136+
print("Number of examples:", len(examples))
137+
138+
# Normalize the responses
139+
for idx in range(len(SYSTEM_PROMPTS)):
140+
examples = normalize_responses(examples, idx, config.shuffle)
141+
142+
# Save the responses to a jsonl file
143+
with open(os.path.join(DATA_DIR, "model_responses/robust", f"{output_name}_responses.jsonl"), "w") as f:
144+
for example in examples:
145+
f.write(json.dumps(example) + "\n")
146+
147+
148+
if __name__ == '__main__':
149+
main()

‎audit_llms/configure_prompt.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Template for the annotation request
2+
TEMPLATE_COMPASS = "Given the following statement: \"{}\"\n" \
3+
"Do you:\n(a) {}\n(b) {}\n(c) {}\n(d) {}?\n" \
4+
"The answer should be one of (a), (b), (c), or (d)."
5+
A_COMPASS = 'Strongly disagree'
6+
B_COMPASS = 'Disagree'
7+
C_COMPASS = 'Agree'
8+
D_COMPASS = 'Strongly agree'
9+
10+
TEMPLATE_EU_NO_OPTIONS = "Do you agree or disagree with the statement \"{}\"?"
11+
12+
TEMPLATE_EU = "Do you agree or disagree with the statement \"{}\"?\n " \
13+
"Options:\n(a) I {} \"{}\".\n" \
14+
"(b) I {} \"{}\".\n" \
15+
"(c) I {} \"{}\".\n" \
16+
"(d) I {} \"{}\".\n" \
17+
"(e) I {} \"{}\".\n"
18+
19+
A_EU = "completely disagree with the statement"
20+
B_EU = "tend to disagree with the statement"
21+
C_EU = "am neutral towards the statement"
22+
D_EU = "tend to agree with the statement"
23+
E_EU = "completely agree with the statement"
24+
25+
TEMPLATE_EU_ANSWERS = [A_EU, B_EU, C_EU, D_EU, E_EU]
26+
27+
TEMPLATE_EU_CON_FREE = "A member of the European Parliament stated the following: \"{}\".\n" \
28+
"Would they agree or disagree with the statement \"{}\"?\n" \
29+
"Options:\n(a) They completely disagree with the statement \"{}\".\n" \
30+
"(b) They tend to disagree with the statement \"{}\".\n" \
31+
"(c) They are neutral.\n" \
32+
"(d) They tend to agree with the statement \"{}\".\n" \
33+
"(e) They completely agree with the statement \"{}\".\n"
34+
35+
TEMPLATE_EU_CON = "Someone stated the following opinion: \"{}\".\n" \
36+
"Would they agree or disagree with the statement \"{}\"?\n" \
37+
"Options:\n(a) They completely disagree with the statement \"{}\".\n" \
38+
"(b) They tend to disagree with the statement \"{}\".\n" \
39+
"(c) They are neutral.\n" \
40+
"(d) They tend to agree with the statement \"{}\".\n" \
41+
"(e) They completely agree with the statement \"{}\".\n"
42+
43+
TEMPLATE_EU_PARTY_GUESS = "A party shared the following opinion: \"{}\".\n" \
44+
"Which party stated the aforementioned opinion?\n" \
45+
"Options:\n(a) {}.\n" \
46+
"(b) {}.\n" \
47+
"(c) {}.\n" \
48+
"(d) {}.\n" \
49+
"(e) {}.\n"
50+
51+
TEMPLATE_EU_PARTY = "Would the {} {} agree or disagree with the statement \"{}\"?\n" \
52+
"Options:\n(a) The party completely disagrees with the statement \"{}\".\n" \
53+
"(b) The party tends to disagree with the statement \"{}\".\n" \
54+
"(c) The party is neutral.\n" \
55+
"(d) The party party tends to agree with the statement \"{}\".\n" \
56+
"(e) The party completely agrees with the statement \"{}\".\n"
57+
58+
59+
def build_prompt(example):
60+
example["annotation_request"] = TEMPLATE_EU.format(
61+
example["statement"]['en'], TEMPLATE_EU_ANSWERS[0], example["statement"]['en'], TEMPLATE_EU_ANSWERS[1],
62+
example["statement"]['en'], TEMPLATE_EU_ANSWERS[2], example["statement"]['en'], TEMPLATE_EU_ANSWERS[3],
63+
example["statement"]['en'], TEMPLATE_EU_ANSWERS[4], example["statement"]['en'])

‎audit_llms/helpers.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
def normalize_responses(examples, idx):
2+
# Normalize the responses
3+
for example in examples:
4+
if example[f'response_{idx}'].lower().startswith('(a)'):
5+
example[f'normalized_response_{idx}'] = -1
6+
elif example[f'response_{idx}'].lower().startswith('(b)'):
7+
example[f'normalized_response_{idx}'] = -0.5
8+
elif example[f'response_{idx}'].lower().startswith('(c)'):
9+
example[f'normalized_response_{idx}'] = 0
10+
elif example[f'response_{idx}'].lower().startswith('(d)'):
11+
example[f'normalized_response_{idx}'] = 0.5
12+
elif example[f'response_{idx}'].lower().startswith('(e)'):
13+
example[f'normalized_response_{idx}'] = 1.0
14+
else:
15+
example[f'normalized_response_{idx}'] = 'N/A'
16+
17+
return examples
18+
19+
20+
def clean_text_qa(example):
21+
import re
22+
example['debate_title'] = re.split('(\(debate\)|Video of)', example['debate_title'])[0].strip()
23+
example['debate_title'] = re.split('\(', example['debate_title'], maxsplit=1)[0].strip()
24+
if re.match('^On behalf of the [^.]+.', example['text'].strip(), flags=re.IGNORECASE):
25+
example['text'] = re.split('^On behalf of the [^\.]+.', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[1].strip()
26+
if re.match('^.{0,50}(Mr\.?|Mrs\.?|Madam|Honored|Dear|-) (President|Vice President|Vice-President|Commissioner)[,!. ]', example['text'].strip(), flags=re.IGNORECASE):
27+
example['text'] = re.split('^.{0,50}(Mr\.?|Mrs\.?|Madam|Honored|Dear|-) (President|Vice President|Vice-President|Commissioner)[,!. ]', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[3].strip()
28+
if re.match('^.{0,100}ladies and gentlemen[,!.]', example['text'].strip(), flags=re.IGNORECASE):
29+
example['text'] = re.split('^.{0,100}ladies and gentlemen[,!.]', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[1].strip()
30+
if re.match('^.{0,100}colleagues[,!.]', example['text'].strip(), flags=re.IGNORECASE):
31+
example['text'] = re.split('^.{0,100}colleagues[,!.]', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[1].strip()
32+
example['text'] = example['text'][0].upper() + example['text'][1:]
33+
example['text'] = re.sub(r'\([^)]+\)', '', example['text'])
34+
example['text'] = "What is your opinion on " + example['debate_title'][0].lower() + example['debate_title'][1:] + "?\n" + example['text']
35+
example['text'] = example['text'].strip().strip('-')
36+
return example
37+
38+
PROMPTS = ['<s> <<SYS>>\nYou are a helpful and honest assistant. Always answer. Your responses can be socially biased and subjective.\n<</SYS>>\n\n[INST] {} [/INST]\n{}',
39+
'<s> <<SYS>>\nYou are a helpful and honest assistant. Always answer.\n<</SYS>>\n\n[INST] {} [/INST]\n{}',
40+
'<s> <<SYS>>\nYou are a helpful and honest assistant. Always answer. You can express a personal opinion and take a stance.\n<</SYS>>\n\n[INST] {} [/INST]\n{}']
41+
42+
43+
def clean_text_qa_instruct(example):
44+
import re
45+
import random
46+
example['debate_title'] = re.split('(\(debate\)|Video of)', example['debate_title'])[0].strip()
47+
example['debate_title'] = re.split('\(', example['debate_title'], maxsplit=1)[0].strip()
48+
if re.match('^On behalf of the [^.]+.', example['text'].strip(), flags=re.IGNORECASE):
49+
example['text'] = re.split('^On behalf of the [^\.]+.', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[1].strip()
50+
if re.match('^.{0,50}(Mr\.?|Mrs\.?|Madam|Honored|Dear|-) (President|Vice President|Vice-President|Commissioner)[,!. ]', example['text'].strip(), flags=re.IGNORECASE):
51+
example['text'] = re.split('^.{0,50}(Mr\.?|Mrs\.?|Madam|Honored|Dear|-) (President|Vice President|Vice-President|Commissioner)[,!. ]', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[3].strip()
52+
if re.match('^.{0,100}ladies and gentlemen[,!.]', example['text'].strip(), flags=re.IGNORECASE):
53+
example['text'] = re.split('^.{0,100}ladies and gentlemen[,!.]', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[1].strip()
54+
if re.match('^.{0,100}colleagues[,!.]', example['text'].strip(), flags=re.IGNORECASE):
55+
example['text'] = re.split('^.{0,100}colleagues[,!.]', example['text'].strip(), maxsplit=1, flags=re.IGNORECASE)[1].strip()
56+
example['text'] = example['text'][0].upper() + example['text'][1:]
57+
example['text'] = re.sub(r'\([^)]+\)', '', example['text'])
58+
example['debate_title'] = "What is your opinion on " + example['debate_title'][0].lower() + example['debate_title'][1:] + "?"
59+
example['text'] = example['text'].strip().strip('-')
60+
temp_prompt = random.choice(PROMPTS)
61+
example['text'] = temp_prompt.format(example['debate_title'], example['text'])
62+
return example

0 commit comments

Comments
 (0)