Spaces:
Sleeping
Sleeping
| import json | |
| import numpy as np | |
| import random | |
| import uuid | |
| def load_from_jsonl(filename, n=np.inf): | |
| data = [] | |
| with open(filename, 'r') as file: | |
| for i, line in enumerate(file): | |
| if i >= n: # stop after reading n lines | |
| break | |
| data.append(json.loads(line)) | |
| return data | |
| def append_id(conversations_no_id): | |
| conversations = [] | |
| for conversation in conversations_no_id: | |
| conversations.append({ | |
| 'conv_id': uuid.uuid4().hex, | |
| 'transcript': conversation['transcript'] | |
| }) | |
| return conversations | |
| def save_to_jsonl(data, filename): | |
| with open(filename, 'w') as file: | |
| for item in data: | |
| json_line = json.dumps(item) | |
| file.write(json_line + '\n') | |
| def get_conversation(conversation_data): | |
| conv = random.choice(conversation_data) | |
| return conv | |
| def pad_transcript(transcript, max_length): | |
| padding_count = max_length - len(transcript) | |
| if padding_count > 0: | |
| for _ in range(padding_count): | |
| transcript.append({'speaker': '', 'response': ''}) | |
| return transcript | |
| def get_last_response(transcript): | |
| for turn in reversed(transcript): | |
| if turn['speaker'] and turn['response']: | |
| return turn['response'] | |