35 lines
834 B
Python
35 lines
834 B
Python
|
import json
|
||
|
|
from tqdm import tqdm
|
||
|
|
import numpy as np
|
||
|
|
import time
|
||
|
|
|
||
|
|
print('start')
|
||
|
|
start_time = time.time()
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
inputfile = open('./data/main_sections_170k_metadata.json', "r", encoding='utf-8')
|
||
|
|
|
||
|
|
data = json.load(inputfile)
|
||
|
|
|
||
|
|
inputfile.close()
|
||
|
|
|
||
|
|
dict = {}
|
||
|
|
|
||
|
|
for item in tqdm(data):
|
||
|
|
key = item['qanon_id']
|
||
|
|
child_order = item['child_order']
|
||
|
|
content = item['content']
|
||
|
|
level =item['other_info']['level']
|
||
|
|
if not key in dict:
|
||
|
|
dict[key] = []
|
||
|
|
dict[key].append({'content':content, 'child_order':child_order, 'level': level})
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
outputfile = open('./data/main_qanon_170k_metadata.json', "w", encoding='utf-8')
|
||
|
|
outputfile.write(json.dumps(dict, ensure_ascii=False, indent = 4))
|
||
|
|
outputfile.close()
|
||
|
|
end_time = time.time()
|
||
|
|
print(f"elapsed time: {end_time-start_time}")
|
||
|
|
print("end")
|