PJMixers-Dev/Nelathan_synthetic-sugar-quill-cleaner

import re import ftfy from datasets import load_dataset from tqdm import tqdm import pandas as pd def process_text(text): # use ftfy text = ftfy.fix_text(text) # unify newline style text = text.replace("\r", "\n") # replace tabs text = text.replace("\t", " ") # replace fancy double quotestext = re.sub(r"[“”]", '"', text) # replace fancy single quotes text = re.sub(r"[‘’]", "'", text) # replace double single quotes with double quotes text =… See the full description on the dataset page: huggingface.co/datasets/PJMixers-Dev/Nelathan_synthetic-sugar-quill-cleaner.