36 lines
1.0 KiB
Python
36 lines
1.0 KiB
Python
from elastic_helper import ElasticHelper
|
||
import datetime
|
||
import json
|
||
|
||
eh_obj = ElasticHelper()
|
||
path = "/home/gpu/data_11/14040423/mj_qa_section.zip"
|
||
sections = eh_obj.iterateJsonFile(path, True)
|
||
|
||
# تاریخی که داده ها از این تاریخ به بعد، پردازش های مختلف را ندیده است
|
||
update_time = datetime.datetime(1403,10,5)
|
||
|
||
def get_data_from_date(date):
|
||
recent_sections = {}
|
||
for i, item in enumerate(sections):
|
||
id = item['id']
|
||
source = item['source']
|
||
ts_date = source['ts_date']
|
||
ts_date_standard = datetime.datetime(ts_date.split('/')[0],ts_date.split('/')[1],ts_date.split('/')[2])
|
||
|
||
if ts_date_standard>date:
|
||
recent_sections[id] = source
|
||
|
||
return recent_sections
|
||
|
||
if __name__ == '__main__':
|
||
recent_sections = get_data_from_date(update_time)
|
||
|
||
with open('./data/recent_sections.json', 'w', encoding='utf-8') as file:
|
||
data = json.dump(recent_sections)
|
||
file.write(data)
|
||
|
||
print('finished!')
|
||
|
||
|
||
|