forked from shibing624/MedicalGPT
-
Notifications
You must be signed in to change notification settings - Fork 0
/
convert_alpaca.py
42 lines (33 loc) · 1.3 KB
/
convert_alpaca.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
"""
Convert alpaca dataset into sharegpt format.
Usage: python convert_alpaca.py --in_file alpaca_data.json --out_file alpaca_data_sharegpt.json
"""
import argparse
from datasets import load_dataset
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--in_file", type=str)
parser.add_argument("--out_file", type=str)
args = parser.parse_args()
data_files = {"train": args.in_file}
raw_datasets = load_dataset('json', data_files=data_files)
def process(examples):
ids = []
convs = []
langs = []
id = 0
for instruction, inp, output in zip(examples['instruction'], examples['input'], examples['output']):
if len(inp.strip()) > 1:
instruction = instruction + '\nInput:\n' + inp
q = instruction
a = output
convs.append([
{"from": "human", "value": q},
{"from": "gpt", "value": a},
])
id += 1
ids.append(f'alpaca_{id}')
langs.append('zh')
return {'id': ids, 'conversations': convs, 'lang': langs}
dataset = raw_datasets['train'].map(process, batched=True, remove_columns=raw_datasets['train'].column_names)
dataset.to_json(f"{args.out_file}", lines=True, force_ascii=False)