ajokar/Nlp_models
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update persian_nlp_model_sqlite --> fastapi

Browse Source
This commit is contained in:
mdorstkar 2025-08-20 06:43:21 +03:30
parent 5513c211df
commit 792e7b6ba1
1 changed files with 374 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

678
persian_nlp_model_sqlite.py
View File

@ -1,65 +1,83 @@
#بسم الله
from apscheduler.schedulers.background import BackgroundScheduler
from transformers import AutoTokenizer
from bidi.algorithm import get_display
from huggingface_hub import HfApi
import matplotlib.pyplot as plt
from fastapi import FastAPI
from datetime import date
import arabic_reshaper
from fpdf import FPDF
import threading
import random
import logging
import sqlite3
import string
import os
# تنظیم لاگ‌دهی برای دیدن خروجی زمان‌بندی
logging.basicConfig(level=logging.INFO)
logging.getLogger('apscheduler').setLevel(logging.INFO)
first_id = 6600
text = 'جمهوری موافقت‌نامه معاملات قانون بودجه اساسی قضائی بین‌المللی تأسیس منطقه‌ای لازم‌الاجراء دامپروری راه‌آهن کمیسیون‌های جدیدالاحداث مسئول فرآورده زائد اسقاط پنجساله'
list1 = ["ID","model_id","url","downloads","private","author","tags","tag_dataset",\
"tag_base_model","tag_license","tag_region","pipeline_tag","Likes","languages",\
"library","datasets","license","just_persian","deleted","date_added","last_modified"]
cnt = sqlite3.connect(".\\db\\persian_nlp_model.db")
cnt = sqlite3.connect(".\\db\\persian_nlp_model.db", check_same_thread=False)
c = cnt.cursor()
today = date.today()
d1 = today.strftime("%d-%m-%Y")
create_tables=True
if create_tables == True :
try:
# فقط برای اولین بار که جدول قرار است ساخته شود از این کد ها استفاده شود
# c.execute("""CREATE TABLE PersianNlp(
# ID INT PRIMARY KEY ,
# model_id TEXT ,
# url TEXT ,
# downloads INT,
# private TEXT,
# author TEXT,
# tags TEXT,
# tag_dataset TEXT,
# tag_base_model TEXT,
# tag_license TEXT,
# tag_region TEXT,
# pipeline_tag TEXT,
# Likes INT,
# languages TEXT,
# library TEXT,
# datasets TEXT,
# license TEXT,
# just_persian TEXT,
# deleted TEXT,
# date_added TEXT,
# last_modified TEXT
# );""")
c.execute("""CREATE TABLE PersianNlp(
ID INT PRIMARY KEY ,
model_id TEXT ,
url TEXT ,
downloads INT,
private TEXT,
author TEXT,
tags TEXT,
tag_dataset TEXT,
tag_base_model TEXT,
tag_license TEXT,
tag_region TEXT,
pipeline_tag TEXT,
Likes INT,
languages TEXT,
library TEXT,
datasets TEXT,
license TEXT,
just_persian TEXT,
deleted TEXT,
date_added TEXT,
last_modified TEXT
);""")
# برای ساخت جدول میزان دانلود ها از این کد استفاده شود
# c.execute("""CREATE TABLE downloadCountHistory(
# ID INT PRIMARY KEY ,
# key_id INT ,
# downloads INT,
# date TEXT
# );""")
c.execute("""CREATE TABLE downloadCountHistory(
ID INT PRIMARY KEY ,
key_id INT ,
downloads INT,
date TEXT
);""")
create_tables = False
except Exception as e:
print("--- یک خطای غیرمنتظره در ساخت تیبل رخ داد ---")
print(f"متن خطا: {e}")
@ -107,6 +125,278 @@ def process_text_for_fpdf(text):
def add_download_count():
count = 1
api = HfApi()
allModel = c.execute(f'''SELECT * FROM "PersianNlp"
WHERE deleted != 'True' ;''')
all_model_id = []
for model in allModel:
all_model_id.append([model[0],model[1]])
for id_ in all_model_id:
# try:
print(count)
count+=1
id_12_digits = generate_random_id(length=12, chars=string.digits)
model_details = api.model_info(repo_id=id_[1])
c.execute(f"""INSERT INTO downloadCountHistory(ID,key_id,downloads,date)
VALUES ({id_12_digits},"{int(id_[0])}","{int(model_details.downloads)}","{str(d1)}");""")
# c.execute(f"""INSERT INTO downloadCountHistory(ID,key_id,downloads,date)
# VALUES ({id_12_digits},"{int(id_[0])}","{int(model_details.downloads)+1}","22-08-2025");""")
cnt.commit()
# except:
# print("Error!!")
# add_download_count()
def MultiModelInfo(limit_number=10):
today = date.today()
date_year = today.strftime("%Y")
date_month = today.strftime("%m")
month = int(date_month)
year = int(date_year)
Models_added_this_month=[]
Models_deleted=[]
all_id_download = []
all_download = []
growth_slope_list_info = []
growth_slope_list = []
model_info = c.execute(f'''SELECT *
FROM PersianNlp''')
for model in model_info:
if int(model[19].split("/")[1]) == month and int(model[19].split("/")[2]) == year :
Models_added_this_month.append(model[1])
if str(model[18]) == "True":
Models_deleted.append(model[1])
all_id_download.append([model[0],model[1],model[3],model[11],model[20]])
listX=[]
for model in all_id_download:
downloadCountHistory = c.execute(f'''SELECT *
FROM downloadCountHistory
WHERE key_id = {model[0]}''')
for models in downloadCountHistory :
if int(models[3].split("-")[1]) == month and int(models[3].split("-")[2]) == year :
model[2]=models[2]
listX.append(model)
all_id_download = listX
for model in all_id_download:
all_download.append(model[2])
all_download.sort(reverse=True)
maximum_download_list = all_download[0:limit_number]
maximum_download_info_list = []
n=0
for DCount in maximum_download_list:
for model in all_id_download:
if DCount == model[2]:
if n < limit_number :
maximum_download_info_list.append(model)
n+=1
# پیدا کردن بیشترین شیب دانلود ها در چند ماه :
for model in all_id_download:
growth_slope = []
DHList =c.execute(f'''SELECT *
FROM "downloadCountHistory"
WHERE key_id = {model[0]}''')
for data in DHList:
growth_slope.append(data[2])
growth_slopee , lenM = find_growth_slope(growth_slope) # به دست آوردن درصد رشد هر مدل
growth_slope_list.append(growth_slopee)
growth_slope_list_info.append([model[1],growth_slopee,lenM,model[2],model[3]])
growth_slope_list.sort(reverse=True)
maximum_growth_slope_list = growth_slope_list[0:limit_number]
maximum_growth_slope_info_list = []
n=0
for DCount in maximum_growth_slope_list:
for model in growth_slope_list_info:
if DCount == model[1]:
if n < limit_number :
maximum_growth_slope_info_list.append(model)
n+=1
# پایان پیدا کردن شیب
model_id_list = []
model_download_count_list = []
for info in maximum_download_info_list:
model_download_count_list.append(info[2])
model_id_list.append(str(info[1]))
listA = []
for x in model_download_count_list:
listA.append(int(x)/1000000)
model_download_count_list = listA
plt.plot(model_id_list,model_download_count_list,marker='o', linestyle='-')
plt.xticks(rotation=30, ha='right', fontsize=10)
plt.xlabel("Model Name", color='blue')
plt.ylabel("Download Count (milion)" ,color='red')
plt.tight_layout()
plt.savefig(f'Top_{limit_number}_download_rate.png', dpi=300)
pdf = FPDF()
pdf.add_page()
pdf.add_font('B Nazanin', '', '.\\fonts\\B Nazanin.ttf', uni=True)
pdf.set_font("Arial", size=16)
# اضافه کردن متن
pdf.multi_cell(0, 10, f"Top {limit_number} Model Information : \n ----------------------------------------------------------------")
pdf.set_font("Arial", size=12)
pdf.ln()
pdf.multi_cell(0, 5, f" Download Rate Chart :")
pdf.ln()
# اضافه کردن عکس
pdf.image(f'Top_{limit_number}_download_rate.png', x=10, y=pdf.get_y() + 5, w=70)
pdf.ln(20) # یک خط فاصله بعد از عکس
# اضافه کردن جدول
pdf.ln(50)
pdf.ln()
pdf.multi_cell(0, 5, f" Download Rate Table :")
pdf.ln()
# سربرگ جدول
for header in ['Count', 'Model_name','Download-rate','Task','Last_modified']:
if header == 'Model_name':
pdf.cell(80, 10, header, 1, 0, 'C')
else:
pdf.cell(40, 10, header, 1, 0, 'C')
pdf.ln()
# ردیف‌های داده
x=1
for row in maximum_download_info_list:
n=0
row[0] = x
x+=1
for item in row:
if n == 1 :
pdf.set_font("Arial", size=6)
pdf.cell(80, 10, f"https://huggingface.co/{str(item)}", 1, 0, 'C')
n+=1
else:
pdf.set_font("Arial", size=10)
pdf.cell(40, 10, str(item), 1, 0, 'C')
n+=1
pdf.ln()
pdf.add_page()
model_name_list = []
model_growth = []
for info in maximum_growth_slope_info_list:
model_name_list.append(info[0])
model_growth.append(round(info[1], 2) )
pdf.ln()
pdf.multi_cell(0, 5, f" Download Growth Chart :")
pdf.ln()
plt.figure(figsize=(8, 6)) # تنظیم اندازه کلی نمودار (عرض و ارتفاع بر حسب اینچ)
plt.bar(model_name_list,model_growth,color='lightgreen', width=0.4)
plt.xticks(rotation=30, ha='right', fontsize=10)
plt.xlabel("Model Name", color='blue')
plt.ylabel("Model Growth (%)" ,color='red')
plt.tight_layout()
plt.savefig(f'Top_{limit_number}_growth_rate.png', dpi=300)
pdf.image(f'Top_{limit_number}_growth_rate.png', x=10, y=pdf.get_y() + 5, w=70)
pdf.ln(80) # یک خط فاصله بعد از عکس
pdf.ln()
pdf.multi_cell(0, 5, f" Download Growth Table :")
pdf.ln()
# سربرگ جدول
for header in [ 'Model_name','Growth-rate','Length-month','Download-rate','Task']:
if header == 'Model_name':
pdf.cell(80, 10, header, 1, 0, 'C')
else:
pdf.cell(30, 10, header, 1, 0, 'C')
pdf.ln()
# ردیف‌های داده
x=1
for row in maximum_growth_slope_info_list:
n=0
x+=1
for item in row:
if n == 0 :
pdf.set_font("Arial", size=6)
pdf.cell(80, 10, f"https://huggingface.co/{str(item)}", 1, 0, 'C')
n+=1
else:
pdf.set_font("Arial", size=6)
pdf.cell(30, 10, str(item), 1, 0, 'C')
n+=1
pdf.ln()
pdf.add_page()
pdf.ln()
pdf.set_font("Arial", size=14)
pdf.multi_cell(0, 5, f"Models added this month :")
pdf.set_font("Arial", size=6)
pdf.ln()
txt=''
n=1
for model_name in Models_added_this_month:
txt +=f"{n} --> {model_name}\n"
n+=1
pdf.multi_cell(0, 5, f"{txt}")
pdf.ln()
pdf.set_font("Arial", size=14)
pdf.multi_cell(0, 5, f"Models deleted :")
pdf.set_font("Arial", size=6)
pdf.ln()
txt=''
n=1
for model_name in Models_deleted:
txt +=f"{n} --> {model_name}\n"
n+=1
pdf.multi_cell(0, 5, f"{txt}")
pdf.output(f"MultiModelInfo_{today}.pdf")
os.remove(f'Top_{limit_number}_download_rate.png')
os.remove(f'Top_{limit_number}_growth_rate.png')
# MultiModelInfo(5)
def find_growth_slope(number_list):
@ -150,12 +440,13 @@ def find_growth_slope(number_list):
def persian_model_finder(nlp_tasks,idx):
today = date.today()
download_date = today.strftime("%d/%m/%Y")
idX = idx # اخرین آیدی موجود در دیتابیس را وارد میکنیم تا موارد جدید با آیدی های قبلی تداخل نکند
api = HfApi()
all_persian_nlp_models_data = []
# all_persian_nlp_models_data = []
seen_model_ids = set() # برای جلوگیری از اضافه شدن مدل‌های تکراری
new_seen_ids = set()
@ -292,6 +583,10 @@ WHERE model_id = '{modelID}';''')
cnt.commit()
add_download_count()
MultiModelInfo()
#اول لیست تسک ها را میدهیم برای جست و جو ، و بعد اولین آیدی که در تیبل مدلها در دیتابیس موجود است
# persian_model_finder(nlp_task_list,6600)
@ -392,35 +687,6 @@ FROM PersianNlp''')
def add_download_count():
count = 1
api = HfApi()
allModel = c.execute(f'''SELECT *
FROM PersianNlp''')
all_model_id = []
for model in allModel:
all_model_id.append([model[0],model[1]])
for id_ in all_model_id:
# try:
print(count)
count+=1
id_12_digits = generate_random_id(length=12, chars=string.digits)
model_details = api.model_info(repo_id=id_[1])
c.execute(f"""INSERT INTO downloadCountHistory(ID,key_id,downloads,date)
VALUES ({id_12_digits},"{int(id_[0])}","{int(model_details.downloads)}","{str(d1)}");""")
# c.execute(f"""INSERT INTO downloadCountHistory(ID,key_id,downloads,date)
# VALUES ({id_12_digits},"{int(id_[0])}","{int(model_details.downloads)+1}","22-08-2025");""")
cnt.commit()
# except:
# print("Error!!")
# add_download_count()
def singleModelInfo( model_id_ ,month_later = 6 , year_later = 0 ):
@ -535,247 +801,51 @@ WHERE key_id = {model_id}''')
def MultiModelInfo(limit_number=10):
today = date.today()
date_year = today.strftime("%Y")
date_month = today.strftime("%m")
month = int(date_month)
year = int(date_year)
Models_added_this_month=[]
Models_deleted=[]
all_id_download = []
all_download = []
growth_slope_list_info = []
growth_slope_list = []
model_info = c.execute(f'''SELECT *
FROM PersianNlp''')
for model in model_info:
if int(model[19].split("/")[1]) == month and int(model[19].split("/")[2]) == year :
Models_added_this_month.append(model[1])
if str(model[18]) == "True":
Models_deleted.append(model[1])
all_id_download.append([model[0],model[1],model[3],model[11],model[20]])
listX=[]
for model in all_id_download:
downloadCountHistory = c.execute(f'''SELECT *
FROM downloadCountHistory
WHERE key_id = {model[0]}''')
for models in downloadCountHistory :
if int(models[3].split("-")[1]) == month and int(models[3].split("-")[2]) == year :
model[2]=models[2]
listX.append(model)
all_id_download = listX
for model in all_id_download:
all_download.append(model[2])
all_download.sort(reverse=True)
maximum_download_list = all_download[0:limit_number]
maximum_download_info_list = []
n=0
for DCount in maximum_download_list:
for model in all_id_download:
if DCount == model[2]:
if n < limit_number :
maximum_download_info_list.append(model)
n+=1
# پیدا کردن بیشترین شیب دانلود ها در چند ماه :
for model in all_id_download:
growth_slope = []
DHList =c.execute(f'''SELECT *
FROM "downloadCountHistory"
WHERE key_id = {model[0]}''')
for data in DHList:
growth_slope.append(data[2])
growth_slopee , lenM = find_growth_slope(growth_slope) # به دست آوردن درصد رشد هر مدل
growth_slope_list.append(growth_slopee)
growth_slope_list_info.append([model[1],growth_slopee,lenM])
growth_slope_list.sort(reverse=True)
maximum_growth_slope_list = growth_slope_list[0:limit_number]
maximum_growth_slope_info_list = []
n=0
for DCount in maximum_growth_slope_list:
for model in growth_slope_list_info:
if DCount == model[1]:
if n < limit_number :
maximum_growth_slope_info_list.append(model)
n+=1
# پایان پیدا کردن شیب
model_id_list = []
model_download_count_list = []
for info in maximum_download_info_list:
model_download_count_list.append(info[2])
model_id_list.append(str(info[1]))
listA = []
for x in model_download_count_list:
listA.append(int(x)/1000000)
model_download_count_list = listA
plt.plot(model_id_list,model_download_count_list,marker='o', linestyle='-')
plt.xticks(rotation=30, ha='right', fontsize=10)
plt.xlabel("Model Name", color='blue')
plt.ylabel("Download Count (milion)" ,color='red')
plt.tight_layout()
plt.savefig(f'Top_{limit_number}_download_rate.png', dpi=300)
pdf = FPDF()
pdf.add_page()
pdf.add_font('B Nazanin', '', '.\\fonts\\B Nazanin.ttf', uni=True)
pdf.set_font("Arial", size=16)
# اضافه کردن متن
pdf.multi_cell(0, 10, f"Top {limit_number} Model Information : \n ----------------------------------------------------------------")
pdf.set_font("Arial", size=12)
pdf.ln()
pdf.multi_cell(0, 5, f" Download Rate Chart :")
pdf.ln()
# اضافه کردن عکس
pdf.image(f'Top_{limit_number}_download_rate.png', x=10, y=pdf.get_y() + 5, w=70)
pdf.ln(20) # یک خط فاصله بعد از عکس
# اضافه کردن جدول
pdf.ln(50)
pdf.ln()
pdf.multi_cell(0, 5, f" Download Rate Table :")
pdf.ln()
# سربرگ جدول
for header in ['Count', 'Model_name','Download-rate','Task','Last_modified']:
if header == 'Model_name':
pdf.cell(80, 10, header, 1, 0, 'C')
else:
pdf.cell(40, 10, header, 1, 0, 'C')
pdf.ln()
# ردیف‌های داده
x=1
for row in maximum_download_info_list:
n=0
row[0] = x
x+=1
for item in row:
if n == 1 :
pdf.set_font("Arial", size=6)
pdf.cell(80, 10, str(item), 1, 0, 'C')
n+=1
else:
pdf.set_font("Arial", size=10)
pdf.cell(40, 10, str(item), 1, 0, 'C')
n+=1
pdf.ln()
model_name_list = []
model_growth = []
for info in maximum_growth_slope_info_list:
model_name_list.append(info[0])
model_growth.append(round(info[1], 2) )
pdf.ln()
pdf.multi_cell(0, 5, f" Download Growth Chart :")
pdf.ln()
plt.figure(figsize=(8, 6)) # تنظیم اندازه کلی نمودار (عرض و ارتفاع بر حسب اینچ)
plt.bar(model_name_list,model_growth,color='lightgreen', width=0.4)
plt.xticks(rotation=30, ha='right', fontsize=10)
plt.xlabel("Model Name", color='blue')
plt.ylabel("Model Growth (%)" ,color='red')
plt.tight_layout()
plt.savefig(f'Top_{limit_number}_growth_rate.png', dpi=300)
pdf.image(f'Top_{limit_number}_growth_rate.png', x=10, y=pdf.get_y() + 5, w=70)
pdf.ln(80) # یک خط فاصله بعد از عکس
pdf.ln()
pdf.multi_cell(0, 5, f" Download Growth Table :")
pdf.ln()
# سربرگ جدول
for header in [ 'Model_name','Growth-rate','Length-month']:
if header == 'Model_name':
pdf.cell(80, 10, header, 1, 0, 'C')
else:
pdf.cell(40, 10, header, 1, 0, 'C')
pdf.ln()
# ردیف‌های داده
x=1
for row in maximum_growth_slope_info_list:
n=0
x+=1
for item in row:
if n == 0 :
pdf.set_font("Arial", size=6)
pdf.cell(80, 10, str(item), 1, 0, 'C')
n+=1
else:
pdf.set_font("Arial", size=10)
pdf.cell(40, 10, str(item), 1, 0, 'C')
n+=1
pdf.ln()
pdf.ln()
pdf.set_font("Arial", size=14)
pdf.multi_cell(0, 5, f"Models added this month :")
pdf.set_font("Arial", size=6)
pdf.ln()
txt=''
n=1
for model_name in Models_added_this_month:
txt +=f"{n} --> {model_name}\n"
n+=1
pdf.multi_cell(0, 5, f"{txt}")
pdf.ln()
pdf.set_font("Arial", size=14)
pdf.multi_cell(0, 5, f"Models deleted :")
pdf.set_font("Arial", size=6)
pdf.ln()
txt=''
n=1
for model_name in Models_deleted:
txt +=f"{n} --> {model_name}\n"
n+=1
pdf.multi_cell(0, 5, f"{txt}")
pdf.output("MultiModelInfo.pdf")
os.remove(f'Top_{limit_number}_download_rate.png')
os.remove(f'Top_{limit_number}_growth_rate.png')
# MultiModelInfo(5)
# --- بخش ادغام با FastAPI و APScheduler ---
app = FastAPI()
scheduler = BackgroundScheduler()
# یک تابع برای دریافت آخرین ID قبل از اجرای job
# def get_last_id():
# try:
# last_id = c.execute("SELECT MAX(ID) FROM PersianNlp").fetchone()[0]
# return last_id if last_id is not None else 0
# except sqlite3.OperationalError:
# return 0
# تعریف کار زمان‌بندی شده: اجرای persian_model_finder در روز آخر هر ماه
def scheduled_job():
idx = first_id
print(f"شروع اجرای کار زمان‌بندی شده. آخرین ID: {idx}")
persian_model_finder(nlp_task_list, idx)
print("کار زمان‌بندی شده با موفقیت به پایان رسید.")
# scheduler.add_job(scheduled_job, 'cron', day='1', hour='0', minute='0')
# scheduler.add_job(scheduled_job, 'interval', minutes=5) # هر بیست دقیقه تابع رو اجرا میکنه که برای تست درست کار کردن هستش و الا کامنت بشه
scheduler.add_job(scheduled_job, 'cron', day='last', hour='0', minute='0') # آخرین روز هرماه رو به عنوان زمان بندی قرار میده
# رویداد startup: شروع زمان‌بندی هنگام روشن شدن سرور
@app.on_event("startup")
def startup_event():
print("رویداد startup: سرور در حال راه‌اندازی است و زمان‌بندی شروع می‌شود.")
scheduler.start()
print("زمان‌بندی فعال شد.")
# رویداد shutdown: خاموش کردن زمان‌بندی هنگام خاموش شدن سرور
@app.on_event("shutdown")
def shutdown_event():
print("رویداد shutdown: سرور در حال خاموش شدن است و زمان‌بندی متوقف می‌شود.")
scheduler.shutdown()
@app.get("/")
def read_root():
return {"message": "FastAPI service is running and the scheduler is active."}
@app.get("/run_monthly_job_now")
def run_job_manually():
print("درخواست برای اجرای دستی کار ماهانه دریافت شد...")
# اجرای کار در یک ترد مجزا برای جلوگیری از مسدود شدن سرور
threading.Thread(target=scheduled_job).start()
return {"message": "Monthly job has been triggered manually."}