add files from Kafi
This commit is contained in:
parent
c93bbd705c
commit
600536794d
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
Data/
|
||||
.env/
|
||||
151
1.extract_verbs_from_PVC.ipynb
Normal file
151
1.extract_verbs_from_PVC.ipynb
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"import json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"this_dir_path = Path().resolve()\n",
|
||||
"data_path = this_dir_path / \"Data\"\n",
|
||||
"verb_path = data_path / \"PVC\" / \"Data\" / \"TXT\" / \"verb.txt\"\n",
|
||||
"processed_past_verb_path = data_path / \"verbs_past_PVC.json\"\n",
|
||||
"processed_present_verb_path = data_path / \"verbs_present_PVC.json\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open(verb_path, \"r\") as f:\n",
|
||||
" lines = f.readlines()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result: dict[str, list] = {}\n",
|
||||
"for line in lines:\n",
|
||||
" verb = line.split(\"@\")[0]\n",
|
||||
" idx = line.split(\"@\")[1]\n",
|
||||
" if isinstance(result.get(idx), list):\n",
|
||||
" result[idx].append(verb)\n",
|
||||
" else:\n",
|
||||
" result[idx] = [verb]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result_simple_past: dict[str, str] = {}\n",
|
||||
"for line in lines:\n",
|
||||
" if line.split(\"@\")[2:] == [\n",
|
||||
" \"ACTIVE\",\n",
|
||||
" \"SINGULAR\",\n",
|
||||
" \"FIRST_PERSON\",\n",
|
||||
" \"POSITIVE\",\n",
|
||||
" \"م\",\n",
|
||||
" \"PAST\",\n",
|
||||
" \"INDICATIVE\",\n",
|
||||
" \"SIMPLE\\n\",\n",
|
||||
" ]:\n",
|
||||
" result_simple_past[line.split(\"@\")[1]] = line.split(\"@\")[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result_simple_past_list = {\n",
|
||||
" key: value[:-1]\n",
|
||||
" for key, value in sorted(result_simple_past.items(), key=lambda x: x[1])\n",
|
||||
"}\n",
|
||||
"with open(processed_past_verb_path, \"w\", encoding=\"utf-8\") as f:\n",
|
||||
" json.dump(result_simple_past_list, f, ensure_ascii=False, indent=4)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result_simple_present: dict[str, str] = {}\n",
|
||||
"for line in lines:\n",
|
||||
" if line.split(\"@\")[2:] == [\n",
|
||||
" \"ACTIVE\",\n",
|
||||
" \"SINGULAR\",\n",
|
||||
" \"FIRST_PERSON\",\n",
|
||||
" \"POSITIVE\",\n",
|
||||
" \"م\",\n",
|
||||
" \"PRESENT\",\n",
|
||||
" \"INDICATIVE\",\n",
|
||||
" \"SIMPLE\\n\",\n",
|
||||
" ]:\n",
|
||||
" result_simple_present[line.split(\"@\")[1]] = line.split(\"@\")[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"result_simple_present_list = {\n",
|
||||
" key: value[:-1]\n",
|
||||
" for key, value in sorted(result_simple_present.items(), key=lambda x: x[1])\n",
|
||||
"}\n",
|
||||
"with open(processed_present_verb_path, \"w\", encoding=\"utf-8\") as f:\n",
|
||||
" json.dump(result_simple_present_list, f, ensure_ascii=False, indent=4)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
168
2.conjugate_PVC_verbs_using_hazm.ipynb
Normal file
168
2.conjugate_PVC_verbs_using_hazm.ipynb
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import hazm\n",
|
||||
"import json\n",
|
||||
"from pathlib import Path"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"this_dir_path = Path().resolve()\n",
|
||||
"data_path = this_dir_path / \"Data\"\n",
|
||||
"past_verb_path = data_path / \"verbs_past_PVC.json\"\n",
|
||||
"present_verb_path = data_path / \"verbs_present_PVC.json\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open(past_verb_path, \"r\") as f:\n",
|
||||
" past_verbs = json.load(f)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"conj = hazm.Conjugation()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['آجیدم', 'آجیدی', 'آجید', 'آجیدیم', 'آجیدید', 'آجیدند']"
|
||||
]
|
||||
},
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"conj.perfective_past(past_verbs[\"1\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"['نآجیدم', 'نآجیدی', 'نآجید', 'نآجیدیم', 'نآجیدید', 'نآجیدند']\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['نیاجیدم', 'نیاجیدی', 'نیاجید', 'نیاجیدیم', 'نیاجیدید', 'نیاجیدند']"
|
||||
]
|
||||
},
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(conj.negative_perfective_past(past_verbs[\"1\"]))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def negation(verb: str):\n",
|
||||
" verb = verb.split()\n",
|
||||
" if verb[-1].startswith(\"آ\"):\n",
|
||||
" verb[-1] = \"نیا\" + verb[-1][1:]\n",
|
||||
" else:\n",
|
||||
" verb[-1] = \"ن\" + verb[-1]\n",
|
||||
" verb = \" \".join(verb)\n",
|
||||
" return verb\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"[negation(verb) for verb in conj.perfective_past(past_verbs[\"1\"])]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['آجیده شدم',\n",
|
||||
" 'آجیده شدی',\n",
|
||||
" 'آجیده شد',\n",
|
||||
" 'آجیده شدیم',\n",
|
||||
" 'آجیده شدید',\n",
|
||||
" 'آجیده شدند']"
|
||||
]
|
||||
},
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"conj.passive_perfective_past(past_verbs[\"1\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# فعلهای متعدی میتوانند مجهول شوند"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
118
3.dadegan_noor.ipynb
Normal file
118
3.dadegan_noor.ipynb
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"from bs4 import BeautifulSoup\n",
|
||||
"from pathlib import Path\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"this_dir_path = Path().resolve()\n",
|
||||
"data_path = this_dir_path / \"Data\"\n",
|
||||
"input_verb_path = data_path / \"verbs_noor.txt\"\n",
|
||||
"htmls_path = data_path / \"htmls_noor\"\n",
|
||||
"output_verb_path = data_path / \"verbs_noor.json\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open(input_verb_path, \"r\") as f:\n",
|
||||
" verbs = [verb[:-1] for verb in f.readlines()]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 3/3 [00:01<00:00, 2.62it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"for verb in tqdm(verbs):\n",
|
||||
" response = requests.get(\"http://search.dadegan.ir/\", params={\"q\": verb})\n",
|
||||
" with open(htmls_path / (verb + \".html\"), \"w\") as f:\n",
|
||||
" f.write(response.text)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 4167/4167 [00:22<00:00, 186.02it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"result = {}\n",
|
||||
"\n",
|
||||
"for verb in tqdm(verbs):\n",
|
||||
" with open(htmls_path / (verb + \".html\"), \"r\") as f:\n",
|
||||
" text = f.read()\n",
|
||||
" soup = BeautifulSoup(text, \"html.parser\")\n",
|
||||
"\n",
|
||||
" stems = soup.findAll(\"td\", {\"class\": \"c3\"})[1:]\n",
|
||||
" past_stem = stems[0].text\n",
|
||||
" present_stem = stems[1].text\n",
|
||||
"\n",
|
||||
" result[verb] = {}\n",
|
||||
"\n",
|
||||
" result[verb][\"past_stem\"] = past_stem\n",
|
||||
" result[verb][\"present_stem\"] = present_stem"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".env",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user