import re

def clean_text_file(input_file, output_file):
    with open(input_file, 'r', encoding='utf-8') as file:
        lines = file.readlines()

    combined_text = ' '.join([line.strip() for line in lines])

    cleaned_text = re.sub(r'\b(B-ORG|I-ORG|O|B-HALFREFERENCE|I-HALFREFERENCE|B-REFERENCE|I-REFERENCE|B-DATE2|I-DATE2)\b', '', combined_text)
    cleaned_text = re.sub(r'\s+', ' ', cleaned_text).strip()

    with open(output_file, 'w', encoding='utf-8') as file:
        file.write(cleaned_text)

input_path = 'test_dataset.txt'
output_path = 'output2.txt'

clean_text_file(input_path, output_path)

print("محتوای فایل تمیز شد و در فایل جدید ذخیره گردید.")
