From f05dcf19fe3ee3ad6c9f9db48dad6e4a558fb216 Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Fri, 27 Feb 2026 09:06:55 +0400 Subject: [PATCH 01/34] Add venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1baaeb9..2cc12c5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ docProps/ media/ word/ *.xml +venv From 6c83a1fa5a350ac0b69cb70ba3b8259b95dbeda3 Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Fri, 27 Feb 2026 09:18:51 +0400 Subject: [PATCH 02/34] Add __pycache__ to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2cc12c5..08b0a01 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ media/ word/ *.xml venv +*__pycache__/ From 5082c09b0810b3e7a9c729f36beca7a372431567 Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Fri, 27 Feb 2026 09:19:21 +0400 Subject: [PATCH 03/34] Add Fastapi example --- app/main.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 8ee4c96..e03f368 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,17 @@ import docs_parser +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/") +def read_root(): + return {"Hello": "World"} + + +@app.get("/items/{item_id}") +def read_item(item_id: int, q: str | None = None): + return {"item_id": item_id, "q": q} # NOTE: все эти точно работают и работают хорошо # print(docs_parser.get_text("parser/assets/text_and_tables.docx")) @@ -6,4 +19,4 @@ # print(docs_parser.get_text("parser/assets/text_tables_png.docx")) # print(docs_parser.get_text("parser/assets/text_from_img.png")) # print(docs_parser.get_text("parser/assets/main.typ")) -print(docs_parser.get_text("parser/assets/main.pdf")) +# print(docs_parser.get_text("parser/assets/main.pdf")) From 0f4f8b9623afa12f4d9fdfbb449274a443d2e5be Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Fri, 27 Feb 2026 09:27:11 +0400 Subject: [PATCH 04/34] Feat(app): add fastapi test router with parser --- app/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index e03f368..9eda2ac 100644 --- a/app/main.py +++ b/app/main.py @@ -9,9 +9,9 @@ def read_root(): return {"Hello": "World"} -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} +@app.get("/items/") +def read_item(source: str): + return {source: docs_parser.get_text(source)} # NOTE: все эти точно работают и работают хорошо # print(docs_parser.get_text("parser/assets/text_and_tables.docx")) From a3c734926224947ee4d93d24b9328c0d592f6c68 Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Sun, 22 Mar 2026 18:33:42 +0400 Subject: [PATCH 05/34] chore: add .venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 08b0a01..e53a560 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ media/ word/ *.xml venv +.venv *__pycache__/ From 91ed56082a243bfe42ef989b2a01e7cf834c6e9e Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Sun, 22 Mar 2026 18:59:00 +0400 Subject: [PATCH 06/34] feat (app): add ParseResponse schema --- app/schemas.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 app/schemas.py diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 0000000..4504b4f --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel + +class ParseResponse(BaseModel): + file_id: str + original_filename: str + parsed_file_path: str | None = None + status: str + From 3192dbb873380156d1fd78608ec29fa358dae6b2 Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Sun, 22 Mar 2026 18:59:19 +0400 Subject: [PATCH 07/34] feat(app): add upload endpoint --- app/main.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 9eda2ac..a8fcb4b 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,8 @@ import docs_parser -from fastapi import FastAPI - +from fastapi import FastAPI, File, UploadFile +from schemas import ParseResponse +import os +import uuid app = FastAPI() @@ -13,6 +15,26 @@ def read_root(): def read_item(source: str): return {source: docs_parser.get_text(source)} +@app.post("/upload", response_model=ParseResponse) +async def upload_file(file: UploadFile, + parse_immediatly: bool = False): + file_id = uuid.uuid4() + file_path = f"uploads/raw/{file_id}_{file.filename}" + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + content = file.read() + with open(file_path, "wb") as f: + f.write(content) + + parsed_path = None + return ParseResponse( + file_id=file_id, + original_filename=file.filename, + parsed_file_path=parsed_path, + status = "parsed" if parse_immediatly else "uploaded" + ) + + # NOTE: все эти точно работают и работают хорошо # print(docs_parser.get_text("parser/assets/text_and_tables.docx")) # print(docs_parser.get_text("parser/assets/some_text.docx")) From f77921dd2c11ec386da752d1353c3683b9f03ae4 Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Sun, 22 Mar 2026 19:06:33 +0400 Subject: [PATCH 08/34] fix(app): fix uuid response and async file read --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index a8fcb4b..14ba2e1 100644 --- a/app/main.py +++ b/app/main.py @@ -18,11 +18,11 @@ def read_item(source: str): @app.post("/upload", response_model=ParseResponse) async def upload_file(file: UploadFile, parse_immediatly: bool = False): - file_id = uuid.uuid4() + file_id = str(uuid.uuid4()) file_path = f"uploads/raw/{file_id}_{file.filename}" os.makedirs(os.path.dirname(file_path), exist_ok=True) - content = file.read() + content = await file.read() with open(file_path, "wb") as f: f.write(content) From 81cea1fb703192d4e914461a94f66b92e8d7248a Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Sun, 22 Mar 2026 19:08:08 +0400 Subject: [PATCH 09/34] chore: add uploads dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e53a560..55fc748 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ word/ venv .venv *__pycache__/ +app/uploads* \ No newline at end of file From fd5d4a4c709b854422017b1c920789335609c6fb Mon Sep 17 00:00:00 2001 From: Anarchy73 Date: Mon, 23 Mar 2026 10:28:07 +0400 Subject: [PATCH 10/34] feat(app): add docker compose with solr --- app/docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 app/docker-compose.yml diff --git a/app/docker-compose.yml b/app/docker-compose.yml new file mode 100644 index 0000000..1382990 --- /dev/null +++ b/app/docker-compose.yml @@ -0,0 +1,6 @@ +services: + solr: + container_name: solr + image: solr:9.1.0 + ports: + - "8983:8983" From 6d2feb00eeb91fb8ed4b168cedbd61699dd5e1ea Mon Sep 17 00:00:00 2001 From: NomenConservandum Date: Sat, 23 May 2026 19:31:13 +0400 Subject: [PATCH 11/34] Implement basic working cycle --- .gitignore | 3 +- README.md | 3 + app/main.py | 93 +++++- front-end/index.html | 705 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 800 insertions(+), 4 deletions(-) create mode 100644 front-end/index.html diff --git a/.gitignore b/.gitignore index 55fc748..3845252 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ word/ venv .venv *__pycache__/ -app/uploads* \ No newline at end of file +app/uploads* +uploads/ \ No newline at end of file diff --git a/README.md b/README.md index 50f05ea..581a7c7 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ sudo pacman -Syu --needed --noconfirm build-essential pkgconf clang llvm \ ```bash # Запускаете .venv pip install maturin + pip install fastapi + pip install python-multipart + pip install uvicorn ``` ## Как запускать? (работа только в .venv окружении) - Билдим rust либу diff --git a/app/main.py b/app/main.py index 14ba2e1..a54426d 100644 --- a/app/main.py +++ b/app/main.py @@ -1,19 +1,45 @@ import docs_parser from fastapi import FastAPI, File, UploadFile +from fastapi.middleware.cors import CORSMiddleware from schemas import ParseResponse import os import uuid + app = FastAPI() +# Configure CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:3000", "http://0.0.0.0:3000"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) @app.get("/") def read_root(): return {"Hello": "World"} - @app.get("/items/") def read_item(source: str): - return {source: docs_parser.get_text(source)} + try: + # Check if file exists + if not os.path.exists(source): + return {"error": f"File not found: {source}"} + + # Use extract_text instead of get_text + # extract_text returns (text_content, images_dict) + text_content, images = docs_parser.extract_text(source) + + return { + "source": source, + "text": text_content, + "has_images": len(images) > 0, + "image_count": len(images) + } + except Exception as e: + print(f"Error parsing file: {e}") + return {"error": f"Failed to parse file: {str(e)}"} @app.post("/upload", response_model=ParseResponse) async def upload_file(file: UploadFile, @@ -27,13 +53,74 @@ async def upload_file(file: UploadFile, f.write(content) parsed_path = None + parsed_text = None + + if parse_immediatly: + try: + # Use extract_text to get the content + text_content, images = docs_parser.extract_text(file_path) + + # Save parsed text to a file + parsed_dir = f"uploads/parsed/{file_id}" + os.makedirs(parsed_dir, exist_ok=True) + parsed_path = f"{parsed_dir}/{file.filename}.txt" + + with open(parsed_path, "w", encoding="utf-8") as f: + f.write(text_content) + + # Also save images if any + if images: + images_dir = f"{parsed_dir}/images" + os.makedirs(images_dir, exist_ok=True) + for (page_num, img_num), img_data in images.items(): + img_path = f"{images_dir}/page_{page_num}_img_{img_num}.png" + with open(img_path, "wb") as f: + f.write(img_data) + + except Exception as e: + print(f"Parse error: {e}") + parsed_path = None + return ParseResponse( file_id=file_id, original_filename=file.filename, parsed_file_path=parsed_path, - status = "parsed" if parse_immediatly else "uploaded" + status="parsed" if parse_immediatly and parsed_path else "uploaded" ) +# Optional: Add an endpoint to convert files to new format +@app.post("/convert/{file_id}") +async def convert_file(file_id: str, new_format: str = "txt"): + """Convert an uploaded file to new format""" + # Find the original file + uploads_dir = "uploads/raw" + original_file = None + + for file in os.listdir(uploads_dir): + if file.startswith(file_id): + original_file = os.path.join(uploads_dir, file) + break + + if not original_file: + return {"error": "File not found"} + + try: + new_path = f"uploads/converted/{file_id}.{new_format}" + os.makedirs(os.path.dirname(new_path), exist_ok=True) + + docs_parser.convert_to_new_format(original_file, new_path) + + return { + "file_id": file_id, + "converted_to": new_format, + "converted_path": new_path + } + except Exception as e: + return {"error": f"Conversion failed: {str(e)}"} + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) # NOTE: все эти точно работают и работают хорошо # print(docs_parser.get_text("parser/assets/text_and_tables.docx")) diff --git a/front-end/index.html b/front-end/index.html new file mode 100644 index 0000000..6912f0b --- /dev/null +++ b/front-end/index.html @@ -0,0 +1,705 @@ + + + + + + + Document Parser - Extract Text from Documents + + + +
+
+

📄 Document Text Extractor

+

Upload documents, images, and PDFs to extract text content

+ +
+
📁
+
Click or drag & drop to upload
+
Supports: PDF, DOCX, PPTX, PNG, JPG, TYP, and more
+ +
+ +
+ + +
+ + + +
+
+
+ +
+

🔍 Extract Text from File

+
+
+ + +
+ + + + +
+
+ + +
+ + + + \ No newline at end of file From e857debead53228c05b44c6041a759871e545a40 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Thu, 28 May 2026 12:54:55 +0400 Subject: [PATCH 12/34] 0.3. Finished I suppose --- .env.local | 2 + .gitignore | 20 ++++--- Dockerfile | 25 +++++++++ README.md | 5 +- app/__init__.py | 1 + app/__pycache__/__init__.cpython-314.pyc | Bin 0 -> 161 bytes app/__pycache__/database.cpython-314.pyc | Bin 0 -> 1051 bytes app/__pycache__/main.cpython-314.pyc | Bin 0 -> 1960 bytes app/__pycache__/models.cpython-314.pyc | Bin 0 -> 2612 bytes app/database.py | 15 ++++++ app/docker-compose.yml | 6 --- app/main.py | 30 ++++++++++- app/models.py | 37 +++++++++++++ docker-compose.yml | 65 +++++++++++++++++++++++ keyword-extraction | 1 + requirements.txt | 14 +++++ scripts/init_db.py | 21 ++++++++ scripts/wait_for_db.py | 24 +++++++++ 18 files changed, 244 insertions(+), 22 deletions(-) create mode 100644 .env.local create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/__pycache__/__init__.cpython-314.pyc create mode 100644 app/__pycache__/database.cpython-314.pyc create mode 100644 app/__pycache__/main.cpython-314.pyc create mode 100644 app/__pycache__/models.cpython-314.pyc create mode 100644 app/database.py delete mode 100644 app/docker-compose.yml create mode 100644 app/models.py create mode 100644 docker-compose.yml create mode 160000 keyword-extraction create mode 100644 requirements.txt create mode 100644 scripts/init_db.py create mode 100644 scripts/wait_for_db.py diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..8e17951 --- /dev/null +++ b/.env.local @@ -0,0 +1,2 @@ +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/docs_db +SOLR_URL=http://localhost:8983/solr/chunks \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3845252..b3b3ee8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,10 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +bin/ +include/ +lib/ +*lib64* target/ -*.so -_rels/ -docProps/ -media/ -word/ -*.xml -venv -.venv -*__pycache__/ -app/uploads* -uploads/ \ No newline at end of file +share/ +.env +pyvenv.cfg +tree.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..75c91fa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Установка системных зависимостей +RUN apt-get update && apt-get install -y gcc libpq-dev && rm -rf /var/lib/apt/lists/* + +# Копирование requirements и установка +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Копирование всего приложения +COPY ./app /app +COPY ./scripts /scripts + +# Создание директории для кэша моделей +RUN mkdir -p /app/model_cache + +# Установка PYTHONPATH +ENV PYTHONPATH=/app + +EXPOSE 8000 + +# Запуск: ждём БД, создаём таблицы, запускаем uvicorn +CMD ["sh", "-c", "python /scripts/wait_for_db.py && python /scripts/init_db.py && uvicorn app.main:app --host 0.0.0.0 --port 8000"] \ No newline at end of file diff --git a/README.md b/README.md index 581a7c7..9b45643 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,7 @@ sudo pacman -Syu --needed --noconfirm build-essential pkgconf clang llvm \ - maturin: ```bash # Запускаете .venv - pip install maturin - pip install fastapi - pip install python-multipart - pip install uvicorn + pip install -r ./requirements.txt ``` ## Как запускать? (работа только в .venv окружении) - Билдим rust либу diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..eeee602 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +# Пустой файл, чтобы Python считал app пакетом \ No newline at end of file diff --git a/app/__pycache__/__init__.cpython-314.pyc b/app/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..803d10d8a1eca06c2847713dd2d2892d8b37230f GIT binary patch literal 161 zcmdPqZr&it@8klS_*AQ}UCGb&FFIi;^?+6AKFTjZA?jMrn#nN}e<+d-9~>DOthEQ;|v4Q?pQpnN*nK-#7@G zqD97V57E3>SsH|LOjEV!c*w=e3UYBRR+hE+xZH_Ml+{5PYdhd|X^9MY-5ME&T-Ul_ zr*pF3XEJJdR5y6jF?H%xErOVtyE+P}+$-HUJ|K zL?~gdC?sHC=Kxha*e=5C+L;M|duZ8Sszi zYpzZ64jRXF9ZuPj;Zzpwyxp8(?gBME9*B5=Q-hhcywPanD~4x?Pc+(S`f8PWC@@Q7 zwj_4GAI&<7^(j5@Eb~xZp;n*iY_~LhPQ5;J$h| z@}LXBBQmZ$jv{zU&^%L%eevg{D5YLP0^!RpN!Vl@=W9RK( z+!V`YLUM*m0*j#WQ=7tEW;5}B1(i@|iA4jaeRWy4aFml+Y*67t*gyi9_^tfdQQNI7 z*6DG!9bF+OFQNG^6GA>h{0m%q1D8I-$afWbw%l&sY_9BiovXb~TdnS|@m(u(Pn5NI lp{0J1P>C_y_&>{|f*B literal 0 HcmV?d00001 diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1e1f31184ffbfab69b66e6f927de5f4d13e5fad GIT binary patch literal 1960 zcmah~O>Epm6rS<_?mB6c-H?VPU8f5zOVln!=}(%FDx0Kjq^OGAKM7JIirAX}s3Fx69A?O7}B@Purf@`)HPH06U1x%4zIQ5n)m3k-#-q`CTtw4-DZ@!)P zy*KZT-^}(lHzg6Y_t1`u+cOCL#eT$1Xn^&f0W6}ok&egF2~2U8VQ!o|!BalO{J3yJ zq+*DLafwRjaSzf(T{?s#A5=asPqxI4H%DFWYcO=B9rZO7Gy&|kO;(Kb#0ERLg`N5j zc4~vIZegpskVE;VhsiK^%BhlNXK#3@pv3P=L3U}M9kkDH`aWPpG8xtn5A>E z*DSEu(+AP9cAR^56{gQ28BM$eeE|B)&{YlfMog%!C?^>u(VoZ!nt|QvLKc@r-LP}) zVLk@~BLdCG`ebsrF)kY{- zae5zucnD>=CdlH!d+CmwL!S(-rgQMFq;t3E{m$%{gP#q4D}3|j@+*gzJNsAL`mZ1R zwe^LYy&w19*t@D`>uUC%+EQ;lxS}3yU|y;h?TE-K(}`m6XKGL zi@dU=v@@K=MNwJOwlQ2t0cGh3F3Q5vfWYuk4t&%w23+GYaB5;G!2`@Bq+%{n+lGs+ z&9&`GhC4YXyP0jRC$!D6Wox@rBT2rtzj;LDYeyyUc~=x+=5CS)9N>Zp@oqHsN!dqQ zZw9dlg{EVj)A<4imy8fda7R$z3pnReAXlJ z(2>jT$v|)&hq@gA)6l(E5R2%(n)#^o>3Dt5Ks_~B7Y5xOFufobW{FXzv(LoEK6)6} z;m|UQ>;Enyo!5oF#uXK2Ai5ba59DZp`9Li-j>oj`mIHNk`1Ejb_|(|+8GT|wwjHW@ zgyuO&*9)W>LWzAj%qxsD%1eq|xf=LoDSKTYqRgyYJHK!Jq2s%bABTSGU*3PFo;q6> z&W5)6ra-;`7qwBfT$WXk^xiESWpkEP=A%=kLdEZkqYKZUyOR-lm=6@ko+g*!A`g#2 zz(Z)^)`n3$AE7c>aY}xf47u!HVt-HG90({@jQ>EL;hX&(B_2pk^-O+6dUcV%kA-Vp zSGulS%Xr5lPQeG)dBk7fK6yCMh-u1435|bZ(0x2NT8bo}FN`DXt;-ogHfi59Jv}UyNPV5cqT{Ali z1XZmMs6xsmRW9kFQjX+Qj`Y;_)N89)YCRRI)Sh~)l&Ggl?VDZOF%k4oF|yyxdoyp| z%=r7>^Qm-76`;lc9$oq;CJ4WCrQ1Lc+V(nhP2rZHizT5bQn47Jkz$laixQQJF&Znz zX}l;?d7Q6_l$2tECW3Laq*4`ZQb{XnR10jeQnHw$sc~UgI4bDzQ9+l>smW*uxfqQ0X=$L$R+>at~r^-r(D~wJ1l;U63eMFx#Un>#e^vZgW@@>j+r{+ z5^PnS%Xpbdm#$rzW$Hz%+F6$qnRNYnsW1xz#RL}$F5Gz0@NgwOUnt`pWqTC{O`$AM zQ5R_BmQ16%NTqV@s8Ei>f{iytG6M0K#J2%u2$W4>HkV-vLXJ_#2r(6*x@-D2A14vI z;~REoCWFwTMLcR-4t89aP9a43w&6VCmnIx~us@s(4hwRUMJ$s@+^X=Xd@ak=46lYw z)G$2no=Z$75!c3S7$J`u)T$tZ!tcG{Q|uu`4ubPGdM^** zo~A3k7}@_?_Y&_!5AXXzYddfo&e@c^i~NPv8%xQ^f8Ss{SfY;KGk;(lXX9OmK(7vo&? zuuWd$>QOHElZD8RQHuh8rNvco@ut> zQG`v4Vv{BCeTp5=avhT39;#$dzA$5X$hd1* z0o4(8$q62nvnRyp8+>%KD>xgxFF|k;jK7=&$?WQz8`G`i$>wxBwQqH1ip*UR`%WJ^lxu`kG?~#>~*jXjI7>WOE*jH z!O?YfYw&pU5_cR~o9Ledvdyj853htL=ZAidUHrbhZT|+Mk6mN0fL(i;qXKh)Z@am= z$DWXR$sR6K0Z!9BJrn2yJ$-P`e8{R@|BxrU`jD>BVLbu)UuChWn(y3Z@e1$y5&j_A z!#&`e$x|1uOy%3;p!~TQ|PD8H_f&+By-hDeL2hkyp9Dg5Hgy-PH^XtsYiB~Pnecr^LjMVa6MWj7^Z^S80L>46(pfNO#uIm z1^6>Ul6C|MnzHlRha)tnEz3je{)p}f(Cv&R#PLUkzXa%l3;zp=6<_%P literal 0 HcmV?d00001 diff --git a/app/database.py b/app/database.py new file mode 100644 index 0000000..cbafc83 --- /dev/null +++ b/app/database.py @@ -0,0 +1,15 @@ +import os +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker +from sqlalchemy.orm import declarative_base + +# Используем localhost вместо postgres (так как запускаем локально) +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/docs_db") + +engine = create_async_engine(DATABASE_URL, echo=True) +AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession) + +Base = declarative_base() + +async def get_db(): + async with AsyncSessionLocal() as session: + yield session \ No newline at end of file diff --git a/app/docker-compose.yml b/app/docker-compose.yml deleted file mode 100644 index 1382990..0000000 --- a/app/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -services: - solr: - container_name: solr - image: solr:9.1.0 - ports: - - "8983:8983" diff --git a/app/main.py b/app/main.py index a54426d..80ced43 100644 --- a/app/main.py +++ b/app/main.py @@ -1,3 +1,30 @@ +from fastapi import FastAPI +from sqlalchemy import text +from app.database import engine +import os + +app = FastAPI(title="Document Search API") + +@app.on_event("startup") +async def startup(): + try: + async with engine.connect() as conn: + # Используем text() для явного указания SQL запроса + result = await conn.execute(text("SELECT 1")) + await conn.commit() + print("✓ Database connection successful") + except Exception as e: + print(f"✗ Database connection failed: {e}") + +@app.get("/") +def root(): + return {"status": "ok", "message": "Backend is running"} + +@app.get("/health") +def health(): + return {"status": "healthy", "database_url": os.getenv("DATABASE_URL", "not set")} + +""" import docs_parser from fastapi import FastAPI, File, UploadFile from fastapi.middleware.cors import CORSMiddleware @@ -91,7 +118,7 @@ async def upload_file(file: UploadFile, # Optional: Add an endpoint to convert files to new format @app.post("/convert/{file_id}") async def convert_file(file_id: str, new_format: str = "txt"): - """Convert an uploaded file to new format""" + # Convert an uploaded file to new format # Find the original file uploads_dir = "uploads/raw" original_file = None @@ -129,3 +156,4 @@ async def convert_file(file_id: str, new_format: str = "txt"): # print(docs_parser.get_text("parser/assets/text_from_img.png")) # print(docs_parser.get_text("parser/assets/main.typ")) # print(docs_parser.get_text("parser/assets/main.pdf")) +""" \ No newline at end of file diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..0147cd5 --- /dev/null +++ b/app/models.py @@ -0,0 +1,37 @@ +from sqlalchemy import Column, String, Integer, DateTime, ForeignKey, JSON, BigInteger +from sqlalchemy.dialects.postgresql import UUID +import uuid +from datetime import datetime +from app.database import Base + +class User(Base): + __tablename__ = "users" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + username = Column(String, unique=True, nullable=False) + hashed_password = Column(String, nullable=False) + role = Column(String, default="user") + +class Document(Base): + __tablename__ = "documents" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + title = Column(String, nullable=False) + author = Column(String) + uploader_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + upload_date = Column(DateTime, default=datetime.utcnow) + last_edited = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + extension = Column(String) + size_bytes = Column(BigInteger) + description = Column(String) + file_path = Column(String) + is_available_to = Column(JSON) # список ID пользователей + +class Chunk(Base): + __tablename__ = "chunks" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + document_id = Column(UUID(as_uuid=True), ForeignKey("documents.id", ondelete="CASCADE")) + chunk_index = Column(Integer) + text = Column(String) + keywords = Column(JSON) # список строк + language = Column(String) + start_char = Column(Integer) + end_char = Column(Integer) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8d14a3c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,65 @@ +services: + postgres: + image: postgres:15 + container_name: docs_postgres + env_file: + - .env + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-docs_db} + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-docs_db}"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - docs_network + + solr: + image: solr:9.1 + container_name: docs_solr + ports: + - "8983:8983" + volumes: + - solr_data:/var/solr + command: + - bash + - -c + - | + precreate-core chunks + cp -r /opt/solr/server/solr/configsets/_default/conf/* /var/solr/data/chunks/conf/ + curl -X POST -H 'Content-type:application/json' --data-binary '{"add-field":{"name":"embedding","type":"dense_vector","indexed":true,"stored":true,"dimensions":312}}' http://localhost:8983/solr/chunks/schema + networks: + - docs_network + + # backend - временно закомментирован для локальной разработки + # backend: + # build: . + # container_name: docs_backend + # ports: + # - "8000:8000" + # depends_on: + # postgres: + # condition: service_healthy + # solr: + # condition: service_started + # env_file: + # - .env + # environment: + # DATABASE_URL: postgresql+asyncpg://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@postgres:5432/${DB_NAME:-docs_db} + # SOLR_URL: http://solr:8983/solr/chunks + # networks: + # - docs_network + +volumes: + postgres_data: + solr_data: + +networks: + docs_network: + driver: bridge \ No newline at end of file diff --git a/keyword-extraction b/keyword-extraction new file mode 160000 index 0000000..48574fa --- /dev/null +++ b/keyword-extraction @@ -0,0 +1 @@ +Subproject commit 48574fa538e5cc308348d29efcb0e29e8d2f62d6 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5420ced --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi +uvicorn[standard] +sqlalchemy +asyncpg +alembic +python-dotenv +python-multipart +passlib[bcrypt] +python-jose[cryptography] +sentence-transformers +nltk +pymorphy3 +numpy +requests \ No newline at end of file diff --git a/scripts/init_db.py b/scripts/init_db.py new file mode 100644 index 0000000..4a15128 --- /dev/null +++ b/scripts/init_db.py @@ -0,0 +1,21 @@ +import asyncio +import sys +import os + +# Добавляем текущую директорию в путь +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Импортируем БД модули +from app.database import engine, Base +from app.models import * + +async def init_db(): + async with engine.begin() as conn: + # Удаляем все таблицы (для чистой инициализации) + await conn.run_sync(Base.metadata.drop_all) + # Создаём заново + await conn.run_sync(Base.metadata.create_all) + print("Tables created successfully") + +if __name__ == "__main__": + asyncio.run(init_db()) \ No newline at end of file diff --git a/scripts/wait_for_db.py b/scripts/wait_for_db.py new file mode 100644 index 0000000..05ba7c5 --- /dev/null +++ b/scripts/wait_for_db.py @@ -0,0 +1,24 @@ +import asyncio +import asyncpg +import os +import sys + +async def wait_for_db(): + db_url = os.getenv("DATABASE_URL") + if not db_url: + print("DATABASE_URL not set") + sys.exit(1) + # преобразуем postgresql+asyncpg:// -> postgresql:// + dsn = db_url.replace("postgresql+asyncpg://", "postgresql://") + while True: + try: + conn = await asyncpg.connect(dsn) + await conn.close() + print("Database is ready") + break + except Exception as e: + print(f"Waiting for database... {e}") + await asyncio.sleep(1) + +if __name__ == "__main__": + asyncio.run(wait_for_db()) \ No newline at end of file From 4257d7206cfd38c6070fa852eb693754636c8113 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Thu, 28 May 2026 12:59:47 +0400 Subject: [PATCH 13/34] 0.4. Finished --- README.md | 3 +- app/__pycache__/embeddings.cpython-314.pyc | Bin 0 -> 1851 bytes app/__pycache__/main.cpython-314.pyc | Bin 1960 -> 2199 bytes app/embeddings.py | 31 +++++++++++++++++++++ app/main.py | 9 ++++-- 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 app/__pycache__/embeddings.cpython-314.pyc create mode 100644 app/embeddings.py diff --git a/README.md b/README.md index 9b45643..09e895d 100644 --- a/README.md +++ b/README.md @@ -30,5 +30,6 @@ sudo pacman -Syu --needed --noconfirm build-essential pkgconf clang llvm \ ``` - Запускаем python ```bash - python main.py + docker-compose up -d postgres solr + uvicorn app.main:app --reload ``` diff --git a/app/__pycache__/embeddings.cpython-314.pyc b/app/__pycache__/embeddings.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0e878e1e4e7a04066b03f67b0f23cadd5ab226ad GIT binary patch literal 1851 zcmah}&2Jk;6rcU@`YUmrCXpc&vXp>o(Z)0_5GsTM0@O%p6_yYoly-6LN#e-fwPwZ@ z?g1@EAF*Ditj`6d?}1r5MyJZ^m95N>s-3+c$6Cd-Fbi z^PWxhCJ-!Ldt_-vN9Z@P376Jpn(qVSp|i+RCecZSDZ_$SCsn4NQKyYiz59IwYaI3~dA%{>-AXK|6 zfEDNrf4~BRU{)heRtq%BphD#mWkF;fl45{iL%=rAs0imE5vc_|ww-qzmqC|on+$-c zAO7?SARfApV#?q>LB7&<#b;Gtedn8`lys8=Rk_;9=}nOW;6F?n9d9dCm04w4r2oL0 zg(qyOFSg&uUazC$L7kN;*OtIVf`17h-Qa7!2jILE!g>u5d&6JmH+^p}ST{^S_$FW( zRD|M3@NJAH_}5^+0gqrEylsIo$ovpZ+xF*uWjKMV{&ErEJ7pH6vi+({2! z?fIf-Wv21^=RMo$mu{yg{Pe_j`ZaIz=i~s_2fA$t%Pq<%F*YJyJse{>1rVf?UQi|b zfIy-eJlm{7WHeo%NN7m%IH*x^D)6By`|b&{+Alw?XyB}~ga?R5Z;FBJL8HZewfA7N zXZJ)a|J#PW`*;X?6E^do^Cepvhg(Su?r0J53|m#t{HyAAu>^C1mRyGngNjT{fY?L7 zCQ~w%Y$6z>?YVEW?%=0#>I6|Sh+y~yaqv6U8cH$ScJdYRm_)`z_>(x1EtdB@7*yDnqG0qWkX=ntl;6?wzo68ec=~pH=%@J5%J_DC#4~mclt_8TA9_^T|JQza Mm-h#t2QuQn0VIg_<^TWy literal 0 HcmV?d00001 diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc index b1e1f31184ffbfab69b66e6f927de5f4d13e5fad..0d0653d198309edb55de8a4533aa31aefdede71b 100644 GIT binary patch delta 1120 zcmZ`&T~E_c7=BN`+jSd=2oq$*O6GvbHov0GkE|{-okpvPMo=v1;2j|q-2COTpm&ZBI|{7E@7R`;ycHMxFNT9y0O$OHe8%d2Yt8g)=-REP%Vf2xrv z)sPCMoH03FGR#7w>$2%gn$s6e!^jsePa64YvuNi_MRyb0hZ@hu5kjXYf-PSyw_%v@ zwbpcIqP@jic^MK^%Sae>P=SZ+bsXb3(m00Q87kJD)kf(}kVgtk$gL6_7$NsLHJHvC zI?PZ9s65I;Q?`@NqbPfPPB@j$rnS>be6H_M&dFWO*`_j8Di+Nt2Rm2n*{LbhwlB>V z=6c;dbZ=x%O}83Uysj1U0Ze09S#5W(5*6Ju^e)Nbun~*L;NGFn_K4((0BP5Lis38V zfJKI96Ze>8Iq_QFzAVD}EpI#2r2w$O+u09{SeHj$#}2=WW$GhG>eA6SO!iwl)_m#Q z^G*&g^G!K^<7J1svLu1rxdez!d=UKgw&8`73tBx8dn?3W2=SFg8Yh0)kMX`Ykv4>G z_iHRXM5%t^5sg{-abGIRR(OcG%otWwFb$b5I>u-9R@0iWgdcZ~;`dStZjOV*jBwb8Ax zIkdr5xr4o=fVGi;SAe@><_gXgYcpQyb<5lUM{?0N_X)SXtYF#*+pf%x5k0gP zK|%B&7xE=~D>8Zz`U`e^4@p7V3bMBzf(-=;gF1ItWU1kv!}p%^x!<|WoaKhsb^c=k zzlY#+k3??PFZr9`{Zdc7Fby5V5DjSvc#jZUo|Z2(RCR4OhaGx1c?miwuPCaOHr*{2WzeZoRKi@hgDxsXwT|mT zpBI_p26+WA1N}(AsrPUlb@ANF&pI8Ivl@S| z)9p=qs1pw)Me2-8=(#3}SXuK>)FtSf!3z^6NEe&KD0cYH;$u}3tX$5vnou^foD_u! zdm^PE$bLu@EgZEXhp04XrZV=NrD7$_&dFU8@8Lk$@`*wl$`Vb@&zF>3_NsZ?%-WSd zQKC1@s( list[float]: + """ + Возвращает вектор эмбеддинга для текста. + Размерность: 312. + """ + model = load_model() + # convert_to_numpy=True возвращает numpy array, .tolist() превращает в list + embedding = model.encode(text, convert_to_numpy=True).tolist() + return embedding + +def get_embedding_dimension() -> int: + """Возвращает размерность эмбеддинга (312 для rubert-tiny2)""" + return 312 \ No newline at end of file diff --git a/app/main.py b/app/main.py index 80ced43..9da30ae 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,9 @@ from sqlalchemy import text from app.database import engine import os +from fastapi import FastAPI +from app.database import engine +from app.embeddings import load_model, get_embedding_dimension app = FastAPI(title="Document Search API") @@ -12,9 +15,11 @@ async def startup(): # Используем text() для явного указания SQL запроса result = await conn.execute(text("SELECT 1")) await conn.commit() - print("✓ Database connection successful") + print("Database connection successful") except Exception as e: - print(f"✗ Database connection failed: {e}") + print(f"Database connection failed: {e}") + load_model() + print(f"Embedding model loaded, dimension: {get_embedding_dimension()}") @app.get("/") def root(): From 52fbd1cde995df76ef607ab6e88fe8d8cd3ea411 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Fri, 29 May 2026 16:07:08 +0400 Subject: [PATCH 14/34] Finish 1.2 --- app/__pycache__/models.cpython-314.pyc | Bin 2612 -> 2879 bytes app/models.py | 19 ++++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/app/__pycache__/models.cpython-314.pyc b/app/__pycache__/models.cpython-314.pyc index f4bf79da67145cd4e9946b940a0119178daa71b4..c74c2570c3c87b017e1bf8ce20c0ea50f63926d1 100644 GIT binary patch delta 1133 zcmZ`%O=uHA6rS0?>}EGf|I*g}B#jnpY^k;s3#Ak-MK=Z08YxAkF56~Fw@r5A?4-6r zNd>`zAjsmyQx9G|*SjYV-dZWNu1F7_1P}fMJPFS1QV600`@Q$Rd2ik~`{s4ao7QT7 zG!h16b#9$n(Lw-xroqV~vA=%)AZ82P1v!`jm|7+SRfu7d`fP?#S&=2RZ@s(=MG zghQ%`MODI*pTlLss*Gh-!HOUAnTQ(2(PS0GK?>xAHjoP~L`T@5ACCE~NLZE~PK77ULdL>I!9X6LvGT@xn(?@~xy+QuPv4x$PP!@hJK9D{J+9&y z=oLh9cYwJr(9@CZR+%{D-RI0W>~!BV*Mu0QV{VeYA)NEw-8J?p9Brh2Gl8q9o!|qJ zp5+XhdDKC|G=t*Ac%p4p%2XZdBw;q)=sTK|AWDD4eEnh5}E(-WL! z6<#WHZt2lXu+_hNAT&WCxRLuc)E5fGwoy;La~7N)`klJCh&%N1)? zGwJUSF2rGD9V5lv6bC#0C2fqh8FSx?w^-xJrkj!yRbMydy;d{7uK7ngT3s)%t&U$c zHtyRf?`Q?n@>l}s3N4jZ9NSvd?6O0iK||SgZ~+<4y`oleOsn8U>lm8T5=?_WcEt;m zJiWNIY?Lio|3(v(qeA`~g(r@e z?0ltYOrR{;h2(QA0^e8&;YT2U0^P@8{t(RXgZX1{iG($dhv&CPYG9VwT2z6(+qedZ zsdY&(xjj__#C&fDP}ma^JN#kK(0|Sh?5>?A7+o@9Y-yR9eJ;L;FrWh I#(w~R0BPd>j{pDw delta 910 zcmbVKT}u>E7(Qpdc6MfW{Yc&NYuf~EElffJ!Mq5`8KD)o6bfn?ok>?`KRSChEFz60 z1VMsx;}7VjYrF3^cvVe^L|5GfEp=nm`_2#;bk$+zd7tyV=gj-ed(PXgHys-nQb~ir z(2moqzf?j#ZoZpC># zvL~L?pQ$d`6DWbM>&LLCj0rZ`sbjOv(dDbb$6a@S|3_EU*6w&3ZyN)7ZaJy8B~^cHnyzjFK|PyEimd2 ze8*o}@yd_FrizP3znC?$8+g^D=9*YP;nZp)a5m1O14oy@WO&wLi^Af6_b$*U%2l`F zdy{M$7E^fZlK@|3O6d_ Date: Sat, 6 Jun 2026 22:40:01 +0400 Subject: [PATCH 15/34] Finish 1 --- app/__pycache__/qdrant_client.cpython-314.pyc | Bin 0 -> 7602 bytes app/document_processor.py | 69 ++++ app/keyword_extraction/README.md | 10 + app/keyword_extraction/__init__.py | 22 + .../collocation_deduplicator.py | 118 ++++++ .../collocation_extractor.py | 195 +++++++++ app/keyword_extraction/example_usage.py | 52 +++ .../example_usage.saving_to_file.py | 153 +++++++ app/keyword_extraction/keyword_processor.py | 147 +++++++ app/keyword_extraction/keyword_set.py | 383 ++++++++++++++++++ app/keyword_extraction/language_detection.py | 61 +++ app/keyword_extraction/lemmatization.py | 62 +++ app/keyword_extraction/paragraph_processor.py | 104 +++++ app/keyword_extraction/requirements.txt | 5 + app/keyword_extraction/stopwords.py | 128 ++++++ app/main.py | 5 + app/qdrant_client.py | 147 +++++++ app/search.py | 8 + docker-compose.yml | 52 +-- requirements.txt | 3 +- test_qdrant.py | 43 ++ 21 files changed, 1728 insertions(+), 39 deletions(-) create mode 100644 app/__pycache__/qdrant_client.cpython-314.pyc create mode 100644 app/document_processor.py create mode 100644 app/keyword_extraction/README.md create mode 100644 app/keyword_extraction/__init__.py create mode 100644 app/keyword_extraction/collocation_deduplicator.py create mode 100644 app/keyword_extraction/collocation_extractor.py create mode 100644 app/keyword_extraction/example_usage.py create mode 100644 app/keyword_extraction/example_usage.saving_to_file.py create mode 100644 app/keyword_extraction/keyword_processor.py create mode 100644 app/keyword_extraction/keyword_set.py create mode 100644 app/keyword_extraction/language_detection.py create mode 100644 app/keyword_extraction/lemmatization.py create mode 100644 app/keyword_extraction/paragraph_processor.py create mode 100644 app/keyword_extraction/requirements.txt create mode 100644 app/keyword_extraction/stopwords.py create mode 100644 app/qdrant_client.py create mode 100644 app/search.py create mode 100644 test_qdrant.py diff --git a/app/__pycache__/qdrant_client.cpython-314.pyc b/app/__pycache__/qdrant_client.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18a7ff1f7b407b3fa928f5d3c03accd8e141b05d GIT binary patch literal 7602 zcmbtZYit`=cD}>m@cpJpy`$07=316w$%$>nv8#u@R{WC3s(@sburxWA7*ixOGt63A zY^o^RZrp6@I9R|Au&BLQ7f8_*(7H*ZX5%dU=psM*W9SHxI#sX%>Oc8MM_Od#{OLJ& zNDd_{c7a|&cg}s^J9Ex=zI(XM<8}~8|1#P3;&%Z;{s#-C#1)0re*gmh zrU_-!u;~!1uvBJ;&4)OJJ7iHTv}_)>D%R(%Bfip9#HjgucRQBaG%ck&t3(WiXW#(+ZR=Q<;RAk|QR~d?YC=nt3Q0SB`1+R3;uvO=e^z zdA$Y(LNgXLb2cL>nk7D&OP`UGyWs$FeZP8cg9l59d9@RcF-z*nc)4{1>NlpH#flO0 z#uh@dWZY7!eTq3@eug9(2x%Zq@cOIC81a$P=z}gowiLTgz$y*ou#=F-;L{xS+wsXB zb;Il3Fsk(O%Oj_7WW|2%rPYn+I8_oXkHeaJ8u64QuW{S59$my%49JjFJWBolGw`eCg4YxNWLL-l=i zK7S!U$EzPf@ptg|f%;MYZ`FnT-}CAEPAZ}=mBUjbv>>zaTzD-;I+Z-$;_14orWwO&!lBhdO4O(P!Qhc&Imd$p*M9niAr}o*Eo!0DoC&aXPHY@GZ+QhV+lf-B| zlS;whWG3CUm)fbkudCb(l}v^+lCR*nYGB~EXXgDEpSmH^X_rQ;&hc#YYk^q8KPVXcfL!?Nkq$%z@)V0n3d%Zz;sAD6_KA|_^h zTliQCDo`nhJnZ|>$WJ*l%N`}|B&Kr~W@y8-Q&7z-I^G=9L#kpk7Y4nseHkO{& z{Q5c-y(Ba}3D2Xs#^@$TVp41h29LpKIW9y`4nKE5vkeswr&$J%pBz4RK(nMW6BDAO znUm?$8I4Pbr*adT{lM9{n8lZ>aiSz;q=-qX!>JjOeovm4{8;oq@DU`05ZsFV2gqhg z!R+DsR>BQS;m&-x^RsZ*Qn)W4?t9(&M{nQ_Igg*@v%tW4`Mmsh<Zr_m-fErRk!R`cOOz~hd*~c^XC<3)f!=7gEr53 z*EyG3y>tG={Gb}zy=2>S$F}FY0t=1z{X#c+HT9z3t&ZrlwaiKZHLtztyYsm5h-WOuK(ZNP@!i31*TJ7z1hrQ-#bLW+<2| z>bY_J&@>sfo%n@{2|%nQ0svO@F1wL+y=~*@tZ3@Cfrqqzg;uNCsB4QR6PgoDGY6Kd z&_b~~oe5McLQ)GN|ExL-iqS077ol(H5qETrVH1O~*{}p7j zp?jFEc>Dzxo_GCK1rDBT77_>*tSH%t(^Igcw3fmP+H{ zX*hD@osX1VsnM|+-1?YBV8$51G-eXmF;*~-nFVf)>jrML3-t+77h|v_aEpc`_Ui4G zxnLV3>z$BbZ-uvMxMy%@V7}natQ9i_E9cNX~%x7Nb)LdEXeyiXHr$(g; zr{>8WfzjyH@_&<`Rp-^~`Sa+dz>86rHU|x}JbJK?u^Kvn=L1J}J^zOK0iXXlEqtWT z8(vS^1`|r1m~F=l%~a}!EW&CgP+E%))|xdNn+7#Y0Ei)P&@9;uD4;y!8Ac5TNT&MC zkIsY*h)!ZtvEjZ6@cCfGL9tB&1_)E+vf#m#3ea_q;G`!NDHm6yX52|DW^I_YLl&_~ zKS25W_AEV)tsRibXp0?sD&nk@ZYN6jDfL4$kNqw~HcQs32p3*7R~JknE`&hlKIdL( z*mlIW_e7qHV{2d4oD=n)J0OXl{RZ^4;g=rG@5&pc;C5(YEgk zZ`F4N4!Ycz@ooL0Kh)n!ZnQcFs@NM{^#fk^rk4YG9Y~^?p;E6-(o;PgBn=Z=2V*7! z=mgdPK*QuJdv~({bY>|6%W2-1a-uXHH2{lKGTG=EX$LeyA2eeL8McHBNu@~(8o%}Mv15@@upWxJ9JMfwOC_h0ist%pQkjevVOi>fbu>{S3TmknX4P&+!GR@PkC{prNP7x0v1=cM3vO1(UV>gQH7HJUWAsk|r zL^+pIfTT~tDNSOJ4(JFN9UVL*ozr;b;x?njDNjHLiW-L*wD!B^5C!xOE^s$cvlM90 z2igJBmjlhqf$HVj#)6#$J5~u7bi8gY)R3B{i?6)>%BtJybKG;0nvS~%6~B1; z!s%sS^`+qD+IMOToZ0OuSV%B@aodG$t5zu8b3pHVIIcjfcE`706A=Fl1jtplMuJ?p zVD{U?K;AD-U6=wEzTDWk9Ijt(Y=-Hp+hF}_&+9+DS3??FVeKjptliN1)*v*7YA>~3 z?s%tTxvKus$;;2a^DL|l{>4J7>n@(Ta0b@)2flVd?{C*|AS~>jEyZa;N8yzMPZ$voEyPQM2*c*M#gCWz+`tCue<(9(+@~r>^^|wOKAq#hl zuN`8zTYDIk`#7k-&2S*!w&2*?wCC+$?NASUyAwyT>3&5)JN~V~kc+%&-Y8BUJ|HRy>Qqmf}P}j?Lv- zt}Izf?Le4)rPGgW;$#Ht1B4kaN1H-CQ!x|98#W|}{~%#@&?wM)!tA6`AQu2Il_~(( ztpo6HC=dQ!nJfd~R?uOf%kKjWf0Ta>X){G>V8{#VhsDSc)g8E5aWBVGISgsAa8uJb zxMamAq1rr^lWFAhbTXxgc+ID1?bWdn;0a?Z8EK|r2qcJW=!_U3HzV<-D}-2NDzNT=E@ zFmVOdHt7~a$ut^NM-g{x^%Zpnkm|Vs`+%F= zbUO$3u{Ue#2llWx_i!M`;UMLN#Fd)#A3%6?2-+}d<4#v$diL_XWkFz!o&xYXFplzF zwj3O-1ORt=Xju?UXdD}YqimM!hhR0B1(hny!kj}jQvU(C$U^=cFcbX_nST`nq7TMM zw}O3u@$f)jfVx2biFz$E6R3#W=}M8P4A7O7xE$QW0G(A=TYpK3%8j}PH^Z)^4T{L< z{vVO46frA~j-^sXHIz<4>uc~YBlWh z(7CQ2a?RP#v)60->)9Li9Ej`O6Iur-2w82TYd_pQbu-|SOu9>=-iV?1p@%u5H(**B zp%opGjq%^Z>{mhh&^-{el~Y+=Z$lMTSA26c;q40(U^ecVriX`DN{X2U@35I)`W|Iv zDncKd1yPtyBKao7@E3so3skC5RyOwxY*l?5wC+Ro8iqX#mDP2%$f-dwuc@D)mVSaH z8#(p02h>?zn5j3A7NW$(07X2LODm|NwhET(Ssz8FN~*bvz)nYOWU?k*eao;}rmJ)& z{FMvh&9IU*0cWCLb_4{}SPSQ0vDGfx>Q}s>C2w2a+jcejnfHm;&CA~4l9$hWdDX_3 z0te8w7;WvD-!b2+hMrus?fSy&qnyouU)B|O2l=hDzlr_0roWE8QOAK8VZK6aiBxIs zbw3P9sK8oaU0v6tqGa`GxJ8-HLJU&B#3K{YtQhjfu%sOhc0><<+oCkU4n`p7%-%L@xo07c@TI5T9bL5d%v!z-)XjRAJ^oqe*JdBn_op_9 cMt(B{Kez<`G{yjj`?}M@^nK$4TcS( List[str]: + """Split a phrase into tokens (words).""" + return phrase.lower().split() + + @staticmethod + def calculate_overlap(phrase1: str, phrase2: str) -> Tuple[int, List[str]]: + """ + Calculate overlapping words between two phrases. + Returns (overlap_count, overlapping_words). + """ + tokens1 = set(CollocationDeduplicator.tokenize_phrase(phrase1)) + tokens2 = set(CollocationDeduplicator.tokenize_phrase(phrase2)) + + overlap = tokens1.intersection(tokens2) + return len(overlap), list(overlap) + + @staticmethod + def is_subsumed(longer: str, shorter: str) -> bool: + """ + Check if shorter phrase is completely contained within longer phrase. + Example: "neural network" is subsumed by "deep neural network" + """ + tokens_longer = CollocationDeduplicator.tokenize_phrase(longer) + tokens_shorter = CollocationDeduplicator.tokenize_phrase(shorter) + + # Check if all tokens of shorter are in longer (in order) + long_idx = 0 + for short_token in tokens_shorter: + found = False + while long_idx < len(tokens_longer): + if tokens_longer[long_idx] == short_token: + found = True + long_idx += 1 + break + long_idx += 1 + if not found: + return False + return True + + + @staticmethod + def remove_overlapping_collocations(collocations: List[Tuple[str, float]], + overlap_threshold: float = 0.6) -> List[Tuple[str, float]]: + """ + Remove overlapping collocations, keeping the most specific (longer) ones. + + Args: + collocations: List of (phrase, score) tuples + overlap_threshold: Minimum overlap ratio to consider as overlapping + (e.g., 0.6 means 60% of words overlap) + + Returns: + Deduplicated list of collocations + """ + if not collocations: + return [] + + # Sort by length (longest first) and then by score + sorted_collocations = sorted(collocations, + key=lambda x: (len(x[0].split()), x[1]), + reverse=True) + + kept = [] + kept_phrases = [] + + for phrase, score in sorted_collocations: + tokens_phrase = set(CollocationDeduplicator.tokenize_phrase(phrase)) + phrase_len = len(tokens_phrase) + + should_keep = True + + # Check against already kept phrases + for kept_phrase in kept_phrases: + tokens_kept = set(CollocationDeduplicator.tokenize_phrase(kept_phrase)) + kept_len = len(tokens_kept) + + # Calculate overlap + overlap_count = len(tokens_phrase.intersection(tokens_kept)) + overlap_ratio = overlap_count / min(phrase_len, kept_len) + + # If significant overlap exists + if overlap_ratio >= overlap_threshold: + # If current phrase is longer, replace the kept one + if phrase_len > kept_len: + # Remove the kept phrase (will be replaced) + continue + else: + # Current phrase is subsumed by kept phrase + should_keep = False + break + + if should_keep: + # Check if this phrase subsumes any kept phrases + new_kept = [] + for kept_phrase in kept_phrases: + if not CollocationDeduplicator.is_subsumed(phrase, kept_phrase): + new_kept.append(kept_phrase) + new_kept.append(phrase) + kept_phrases = new_kept + kept.append((phrase, score)) + + # Sort by score again for output + kept.sort(key=lambda x: x[1], reverse=True) + return kept \ No newline at end of file diff --git a/app/keyword_extraction/collocation_extractor.py b/app/keyword_extraction/collocation_extractor.py new file mode 100644 index 0000000..95e4a03 --- /dev/null +++ b/app/keyword_extraction/collocation_extractor.py @@ -0,0 +1,195 @@ +""" +Collocation extraction utilities for multilingual text. +""" + +import re +from collections import Counter +from typing import List, Tuple, Set +from .stopwords import get_stopwords, is_stopword +from .lemmatization import is_content_word +from .collocation_deduplicator import CollocationDeduplicator + +# Try to import NLTK +try: + import nltk + from nltk.collocations import BigramCollocationFinder, TrigramCollocationFinder + from nltk.metrics import BigramAssocMeasures, TrigramAssocMeasures + + nltk_data_downloaded = False + try: + nltk.data.find('tokenizers/punkt') + nltk.data.find('corpora/stopwords') + nltk_data_downloaded = True + except LookupError: + print("Downloading required NLTK data...") + try: + nltk.download('punkt', quiet=True) + nltk.download('punkt_tab', quiet=True) + nltk.download('stopwords', quiet=True) + nltk_data_downloaded = True + print("NLTK data downloaded successfully") + except Exception as e: + print(f"Could not download NLTK data: {e}") + nltk_data_downloaded = False + + NLTK_AVAILABLE = nltk_data_downloaded + +except ImportError: + print("NLTK not installed. Using fallback methods.") + NLTK_AVAILABLE = False + nltk = None + + +class CollocationExtractor: + """Extracts and processes collocations from text.""" + + def __init__(self, max_ngram_size: int = 3, deduplicator=None): + self.max_ngram_size = max_ngram_size + self.deduplicator = deduplicator or CollocationDeduplicator() + + def extract_collocations_fallback(self, text: str, language: str = 'en', + top_n: int = 10, verbose: bool = False) -> List[Tuple[str, float]]: + """Fallback method for extracting collocations.""" + from .stopwords import get_punctuation_pattern + punct_pattern = get_punctuation_pattern(language) + clean_text = re.sub(punct_pattern, ' ', text.lower()) + words = clean_text.split() + + collocations = [] + + # Стоп-слова, которые не должны быть в коллокациях + stopwords = get_stopwords(language) + bad_patterns = {'это', 'он', 'она', 'оно', 'они', 'этот', 'эта', 'это', 'эти'} + + for n in range(2, self.max_ngram_size + 1): + for i in range(len(words) - n + 1): + ngram_words = words[i:i+n] + ngram = ' '.join(ngram_words) + + # Пропускаем коллокации, начинающиеся с местоимений или предлогов + first_word = ngram_words[0] + if first_word in stopwords or first_word in bad_patterns: + continue + + # Пропускаем коллокации, заканчивающиеся на предлоги + last_word = ngram_words[-1] + if last_word in {'о', 'об', 'в', 'на', 'для', 'с', 'к', 'у', 'по', 'за'}: + continue + + # Проверяем, что коллокация содержит хотя бы одно знаменательное слово + has_content = False + for w in ngram_words: + if is_content_word(w) if language == 'ru' else len(w) > 3: + has_content = True + break + + if not has_content: + continue + + # Считаем частоту + freq = sum(1 for j in range(len(words) - n + 1) + if ' '.join(words[j:j+n]) == ngram) + + if freq >= 1: + # Вес коллокации: частота + длина + score = freq * len(ngram) / 50 + + # Бонус за длину + if n == 2: + score *= 1.2 + elif n == 3: + score *= 1.5 + + collocations.append((ngram, score)) + + # Удаляем дубликаты + unique_collocations = {} + for colloc, score in collocations: + if colloc not in unique_collocations or score > unique_collocations[colloc]: + unique_collocations[colloc] = score + + # Отладочный вывод + if verbose: + print(f"\n[DEBUG] Filtered collocations (after cleaning):") + for colloc, score in list(unique_collocations.items())[:15]: + if verbose: + print(f" - '{colloc}' (score: {score:.3f})") + + sorted_collocations = sorted(unique_collocations.items(), + key=lambda x: x[1], reverse=True) + + return sorted_collocations[:top_n] + + def extract_collocations_nltk(self, text: str, language: str = 'en', + top_n: int = 10) -> List[Tuple[str, float]]: + """Extract collocations using NLTK.""" + if not NLTK_AVAILABLE: + return self.extract_collocations_fallback(text, language, top_n) + + try: + from nltk.corpus import stopwords + + try: + nltk_stopwords = set(stopwords.words(language)) + except: + nltk_stopwords = get_stopwords(language) + + tokens = nltk.word_tokenize(text.lower()) + + filtered_tokens = [token for token in tokens + if token.isalnum() and token not in nltk_stopwords and len(token) > 2] + + if len(filtered_tokens) < 2: + return [] + + collocations = [] + + if self.max_ngram_size >= 2: + try: + bigram_finder = BigramCollocationFinder.from_words(filtered_tokens) + bigram_finder.apply_freq_filter(1) + bigrams = bigram_finder.nbest(BigramAssocMeasures.pmi, min(top_n, 20)) + for bigram in bigrams: + collocation = ' '.join(bigram) + score = len(collocation) / 50 + collocations.append((collocation, score)) + except: + pass + + if self.max_ngram_size >= 3: + try: + trigram_finder = TrigramCollocationFinder.from_words(filtered_tokens) + trigram_finder.apply_freq_filter(1) + trigrams = trigram_finder.nbest(TrigramAssocMeasures.pmi, min(top_n, 20)) + for trigram in trigrams: + collocation = ' '.join(trigram) + score = len(collocation) / 50 + collocations.append((collocation, score)) + except: + pass + + unique_collocations = {} + for colloc, score in collocations: + if colloc not in unique_collocations or score > unique_collocations[colloc]: + unique_collocations[colloc] = score + + sorted_collocations = sorted(unique_collocations.items(), + key=lambda x: x[1], reverse=True) + + # Apply deduplication + deduplicated = self.deduplicator.remove_overlapping_collocations( + sorted_collocations, 0.6 + ) + + return deduplicated[:top_n] + + except Exception as e: + return self.extract_collocations_fallback(text, language, top_n) + + def extract_collocations(self, text: str, language: str = 'en', + top_n: int = 10, verbose: bool = False) -> List[Tuple[str, float]]: + """Extract collocations from text with deduplication.""" + if NLTK_AVAILABLE: + return self.extract_collocations_nltk(text, language, top_n) + else: + return self.extract_collocations_fallback(text, language, top_n, verbose) \ No newline at end of file diff --git a/app/keyword_extraction/example_usage.py b/app/keyword_extraction/example_usage.py new file mode 100644 index 0000000..2004a7c --- /dev/null +++ b/app/keyword_extraction/example_usage.py @@ -0,0 +1,52 @@ +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +os.environ['TRANSFORMERS_OFFLINE'] = '1' +os.environ['HF_HUB_OFFLINE'] = '1' +os.environ['HF_DATASETS_OFFLINE'] = '1' + +from keyword_extraction import MultilingualKeywordSet + +model_cache_path = './model_cache2' + +def enter_example(): + os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1' + os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '1' + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + # Это длинный абзац о машинном обучении. Он охватывает несколько тем, включая нейронные сети, глубокие нейронные сети и сверточные нейронные сети. Цель состоит в том, чтобы извлечь значимые ключевые слова, такие как алгоритмы машинного обучения и методы глубокого обучения, из длинных текстов. + # Современные технологии искусственного интеллекта и машинного обучения стремительно трансформируют различные отрасли промышленности и науки. Глубокие нейронные сети, которые представляют собой многослойные архитектуры с десятками и даже сотнями слоёв, демонстрируют впечатляющие результаты в распознавании образов, обработке естественного языка и компьютерном зрении. Свёрточные нейронные сети, впервые предложенные Яном Лекуном в конце двадцатого века, стали основой для большинства современных систем компьютерного зрения, включая распознавание лиц, автоматическую аннотацию изображений и диагностику медицинских снимков. Рекуррентные нейронные сети и их более совершенные варианты, такие как LSTM и GRU, успешно применяются для анализа последовательных данных, включая временные ряды, тексты на естественном языке и геномные последовательности. Методы глубокого обучения с подкреплением позволили создать алгоритмы, которые превосходят человека в таких сложных играх, как го, шахматы и различные видеоигры от компании Blizzard. Трансформеры, архитектура которых была представлена в знаменитой статье Attention Is All You Need, произвели настоящую революцию в области обработки естественного языка, что привело к созданию таких моделей, как BERT, GPT, T5 и многих других. Эти модели используют механизм внимания, который позволяет эффективно обрабатывать длинные последовательности и улавливать сложные зависимости между элементами текста. + # В последние годы наблюдается значительный прогресс в области квантовых вычислений и их потенциального применения для решения сложных оптимизационных задач. Квантовые процессоры, основанные на сверхпроводящих кубитах и захваченных ионах, постепенно набирают вычислительную мощность, приближаясь к так называемому квантовому превосходству. Исследователи из Google впервые продемонстрировали квантовое превосходство в 2019 году, когда их 53-кубитный процессор Sycamore выполнил специфическую задачу за 200 секунд, на решение которой у самого мощного классического суперкомпьютера потребовалось бы около десяти тысяч лет. Однако критики отмечают, что эта задача была специально подобрана для квантового компьютера и не имеет практического применения. Тем не менее, учёные активно работают над созданием отказоустойчивых квантовых компьютеров с коррекцией ошибок, что является одним из главных препятствий на пути к практическому использованию квантовых алгоритмов. Параллельно развиваются и гибридные подходы, сочетающие классические и квантовые вычисления, которые уже сейчас могут применяться для оптимизации логистических цепочек, моделирования молекулярных структур в фармацевтике и решения других прикладных задач. + # Цифровая трансформация бизнеса в эпоху четвёртой промышленной революции требует от компаний не только внедрения новых технологий, но и пересмотра фундаментальных бизнес-моделей и организационных структур. Большие данные, аналитика в реальном времени и предиктивное моделирование становятся ключевыми инструментами для принятия стратегических решений в условиях высокой неопределённости и турбулентности рынков. Пандемия коронавируса значительно ускорила процессы цифровизации во всех секторах экономики, от розничной торговли и образования до здравоохранения и государственного управления. Компании, которые раньше игнорировали необходимость развития онлайн-каналов продаж и удалённых форматов работы, оказались в крайне уязвимом положении. В то же время лидеры цифровой экономики, такие как Amazon, Microsoft и китайские технологические гиганты, смогли не только сохранить, но и значительно нарастить свои рыночные позиции. Интересно отметить, что внедрение технологий искусственного интеллекта и роботизации процессов пока что приводит к созданию новых рабочих мест и трансформации существующих профессий, а не к массовой безработице, как прогнозировали некоторые пессимистичные эксперты в начале двухтысячных годов. + # Прорывы в области генной инженерии и персонализированной медицины открывают беспрецедентные возможности для лечения ранее неизлечимых заболеваний. Технология редактирования генома CRISPR-Cas9, открытая Эммануэль Шарпантье и Дженнифер Дудной, позволяет вносить точные изменения в ДНК живых организмов, открывая путь к исправлению генетических мутаций, вызывающих наследственные заболевания. Уже проводятся клинические испытания методов терапии серповидноклеточной анемии, муковисцидоза и некоторых форм наследственной слепоты с использованием технологий генного редактирования. Параллельно развиваются методы иммунотерапии рака, включая CAR-T-клеточную терапию, при которой иммунные клетки пациента генетически модифицируются для более эффективного распознавания и уничтожения злокачественных опухолей. В онкологии также активно применяются алгоритмы машинного обучения для анализа медицинских изображений, что позволяет на ранних стадиях выявлять злокачественные новообразования и значительно повышает точность диагностики. Модели глубокого обучения способны различать доброкачественные и злокачественные образования на маммограммах и компьютерных томограммах с точностью, сопоставимой или даже превосходящей опытных радиологов. + # Глобальное изменение климата и антропогенное воздействие на окружающую среду становятся одними из главных вызовов двадцать первого века, требующих безотлагательных и скоординированных действий на международном уровне. Концентрация парниковых газов в атмосфере достигла рекордных значений за последние восемьсот тысяч лет, что приводит к повышению средней температуры на планете, таянию ледников и учащению экстремальных погодных явлений. Участившиеся лесные пожары в Австралии и Сибири, разрушительные ураганы в Атлантике, наводнения в Европе и Юго-Восточной Азии напрямую связаны с климатическими изменениями. Учёные предупреждают, что если не принять решительных мер по сокращению выбросов углекислого газа и других парниковых газов, последствия будут катастрофическими для экосистем и человеческой цивилизации. В ответ на эту угрозу многие страны взяли на себя обязательства по достижению углеродной нейтральности к середине столетия, что предполагает масштабный переход к возобновляемым источникам энергии, электрификацию транспорта и повышение энергоэффективности промышленности и строительства. + # И конечно же никто никогда не мог предположить, что старые методы и подходы окажутся полностью бесполезными в новой реальности. Однако тот самый наш опыт показывает, что не всё так просто и однозначно в этом вопросе. Мы долго обсуждали эту проблему и в итоге пришли к выводу, что нужно действовать незамедлительно. Сначала они попытались решить всё с помощью обычных алгоритмов, но потом поняли, что это был довольно наивный подход. В конце концов, как говорится, настоящие профессионалы всегда ищут нестандартные решения и не боятся экспериментировать. Именно поэтому было принято решение об использовании самых современных нейросетевых архитектур для обработки поступающих данных. + # This is a longer paragraph about machine learning. It covers multiple topics including neural networks, deep neural networks, and convolutional neural networks. The goal is to extract meaningful keywords like machine learning algorithms and deep learning methods from longer texts. + keyword_set = MultilingualKeywordSet( + initial_keywords={ + 'en': [], + 'ru': [] + }, + similarity_threshold=0.75, + collocation_overlap_threshold=0.6, + cache_folder=model_cache_path, + embedding_model='cointegrated/rubert-tiny2', + use_collocations=True, + max_chunk_size=500 + ) + + print("Enter your text: ") + s = input() + + result = keyword_set.process_text(s) + + if 'keywords' in result and result['keywords']: + for i, word in enumerate(result['keywords']): + print(f"{i + 1}: {word}") + else: + print("No keywords found.") + +if __name__ == "__main__": + enter_example() \ No newline at end of file diff --git a/app/keyword_extraction/example_usage.saving_to_file.py b/app/keyword_extraction/example_usage.saving_to_file.py new file mode 100644 index 0000000..3ae6a96 --- /dev/null +++ b/app/keyword_extraction/example_usage.saving_to_file.py @@ -0,0 +1,153 @@ +""" +Example usage of the Multilingual Keyword Set. +""" + +import sys +import os + +# Add parent directory to path if running directly +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from keyword_extraction import MultilingualKeywordSet + +model_cache_path = './model_cache' + +def example_usage(): + os.environ['TRANSFORMERS_OFFLINE'] = '1' + print("MULTILINGUAL SEMANTIC KEYWORD SET (No langdetect)") + print("=" * 70) + + initial_keywords = { + 'en': ["machine learning", "neural network", "deep learning"], + 'ru': ["машинное обучение", "нейронная сеть", "глубокое обучение"] + } + + keyword_set = MultilingualKeywordSet( + initial_keywords=initial_keywords, + similarity_threshold=0.75, + collocation_overlap_threshold=0.6, + cache_folder=model_cache_path, + use_collocations=True, + max_chunk_size=500 + ) + + print("\nInitial keyword set:") + keyword_set.display_keyword_set() + + # Test texts with overlapping phrases + texts = [ + # English sentences + "Deep neural networks are revolutionizing artificial intelligence", + "Глубокие нейронные сети революционизируют искусственный интеллект", + "Support vector machines are used for classification tasks", + "Методы опорных векторов используются для задач классификации", + + # Long English paragraph with overlapping phrases + """ + This is a longer paragraph about machine learning. + It covers multiple topics including neural networks, deep neural networks, + and convolutional neural networks. The goal is to extract meaningful keywords + like machine learning algorithms and deep learning methods from longer texts. + """, + + # Long Russian paragraph with overlapping phrases + """ + Это длинный абзац о машинном обучении. + Он охватывает несколько тем, включая нейронные сети, глубокие нейронные сети + и сверточные нейронные сети. Цель состоит в том, чтобы извлечь значимые ключевые слова, + такие как алгоритмы машинного обучения и методы глубокого обучения, из длинных текстов. + """, + + # Another English text with more overlapping collocations + """ + Convolutional neural networks (CNNs) are a type of neural network architecture. + These deep learning models have revolutionized computer vision tasks. + Transfer learning allows reusing pre-trained neural networks for new tasks. + """ + ] + + print("\n" + "=" * 70) + print("PROCESSING TEXTS WITH OVERLAPPING COLLOCATIONS") + print("=" * 70) + + for i, text in enumerate(texts, 1): + print(f"\n{'#'*70}") + print(f"Text {i}/{len(texts)}") + print(f"{'#'*70}") + keyword_set.process_text(text, top_n=6, verbose=True) + + print("\n" + "=" * 70) + print("FINAL RESULTS") + print("=" * 70) + + # Show final keyword set with frequencies + keyword_set.display_keyword_set(show_frequencies=True) + + # Save keywords to file + keyword_set.save_keywords_to_file('./multilingual_keywords.txt') + """ + # Optional: Show the saved keywords + print("\n" + "=" * 70) + print("SAVED KEYWORDS PREVIEW") + print("=" * 70) + with open('multilingual_keywords.txt', 'r', encoding='utf-8') as f: + lines = f.readlines() + for line in lines[:20]: # Show first 20 lines + print(line.rstrip()) + if len(lines) > 20: + print(f"... and {len(lines) - 20} more lines") + """ + +def simple_example(): + os.environ['TRANSFORMERS_OFFLINE'] = '1' + """Very simple example for quick testing.""" + print("\n" + "=" * 70) + print("SIMPLE QUICK TEST") + print("=" * 70) + + keyword_set = MultilingualKeywordSet( + similarity_threshold=0.7, + collocation_overlap_threshold=0.6, + cache_folder='./simple_cache' + ) + + # Quick English test + keyword_set.process_text("Machine learning is transforming technology", top_n=3) + + # Quick Russian test + keyword_set.process_text("Искусственный интеллект меняет мир", top_n=3) + + # Display results + keyword_set.display_keyword_set() + +def enter_example(): + os.environ['TRANSFORMERS_OFFLINE'] = '1' + # Это длинный абзац о машинном обучении. Он охватывает несколько тем, включая нейронные сети, глубокие нейронные сети и сверточные нейронные сети. Цель состоит в том, чтобы извлечь значимые ключевые слова, такие как алгоритмы машинного обучения и методы глубокого обучения, из длинных текстов. + + keyword_set = MultilingualKeywordSet( + initial_keywords={ + 'en': [], + 'ru': [] + }, + similarity_threshold=0.75, + collocation_overlap_threshold=0.6, + cache_folder=model_cache_path, + use_collocations=True, + max_chunk_size=500 + ) + + print("Enter your text: ") + s = input() + + keyword_set.process_text(s, verbose=True) + + keyword_set.display_keyword_set() + + keyword_set.save_keywords_to_file('./entered_text_keywords.txt') + + +if __name__ == "__main__": + # Run the main example + # example_usage() + + enter_example() \ No newline at end of file diff --git a/app/keyword_extraction/keyword_processor.py b/app/keyword_extraction/keyword_processor.py new file mode 100644 index 0000000..e7d07f8 --- /dev/null +++ b/app/keyword_extraction/keyword_processor.py @@ -0,0 +1,147 @@ +""" +Keyword processing utilities for extraction and filtering. +""" + +import re +from collections import Counter +from typing import List, Tuple, Set +from .stopwords import get_punctuation_pattern, is_stopword +from .lemmatization import lemmatize_russian, lemmatize_russian_collocation, is_function_word +from .language_detection import detect_language_simple + + +class KeywordProcessor: + """Processes keywords: extraction, lemmatization, filtering.""" + + def __init__(self, keybert_model=None): + self.keybert = keybert_model + + def lemmatize_if_necessary(self, keyword: str) -> str: + """Lemmatize keyword if it's Russian.""" + if detect_language_simple(keyword).lower() == 'ru': + return lemmatize_russian_collocation(keyword) + return keyword + + def filter_stopwords(self, words: List[str]) -> List[str]: + """Filter out stopwords from a list of words.""" + filtered = [] + for w in words: + lang = detect_language_simple(w).lower() + lemmatized = self.lemmatize_if_necessary(w) + if not is_stopword(lemmatized, lang) and len(w) > 2: + filtered.append(w) + return filtered + + def extract_simple_keywords(self, text: str, language: str = 'en') -> List[str]: + """Simple keyword extraction as fallback.""" + punct_pattern = get_punctuation_pattern(language) + clean_text = re.sub(punct_pattern, ' ', text.lower()) + words = clean_text.split() + + keywords = self.filter_stopwords(words) + + unique_keywords = [] + seen = set() + for kw in keywords: + if kw not in seen: + seen.add(kw) + unique_keywords.append(kw) + + return unique_keywords[:10] + + def prepare_keyword_for_addition(self, keyword: str, language: str) -> str: + """Prepare keyword before adding to set (lemmatize, normalize).""" + keyword_lower = keyword.lower().strip() + + if language == 'ru': + keyword_lower = lemmatize_russian_collocation(keyword_lower) + + return keyword_lower + + def should_skip_keyword(self, keyword: str, language: str, + existing_keywords: set = None) -> Tuple[bool, str]: + """ + Check if keyword should be skipped. + + Returns: + (should_skip, reason) + """ + keyword_lower = keyword.lower().strip() + + # Check length + if len(keyword_lower) < 3: + return True, "too short" + + # Check if single word is part of existing collocation + if existing_keywords and len(keyword_lower.split()) == 1: + for existing in existing_keywords: + if len(existing.split()) >= 2 and keyword_lower in existing.split(): + return True, f"part of collocation '{existing}'" + + # Check stopwords + words = keyword_lower.split() + stopword_count = sum(1 for w in words if is_stopword(w, language)) + if len(words) > 0 and stopword_count == len(words): + return True, "all words are stopwords" + + # Check function words for Russian + if language == 'ru': + func_count = sum(1 for w in words if is_function_word(w)) + if len(words) > 0 and func_count == len(words): + return True, "all words are function words" + + return False, "" + + def extract_keywords_from_text(self, text: str, language: str = 'en', + top_n: int = 10, verbose: bool = False) -> List[Tuple[str, float]]: + """Extract keywords and collocations from text.""" + all_candidates = [] + from .collocation_extractor import CollocationExtractor + + # Extract collocations + coll_extractor = CollocationExtractor(max_ngram_size=3) + collocations = coll_extractor.extract_collocations(text, language, top_n=top_n * 2, verbose=verbose) + + # Lemmatize collocations + if language == 'ru': + lemmatized_collocs = [] + for phrase, score in collocations: + lemmatized_phrase = lemmatize_russian_collocation(phrase) + words = lemmatized_phrase.split() + if len(words) >= 2: + lemmatized_collocs.append((lemmatized_phrase, score * 2.0)) + if verbose: + print(f"[COLLOC] Found: '{phrase}' -> '{lemmatized_phrase}' (score: {score:.3f})") + all_candidates.extend(lemmatized_collocs) + else: + all_candidates.extend(collocations) + + # Extract single words using simple method (without KeyBERT) + simple_keywords = self.extract_simple_keywords(text, language) + + # Filter words that are already in collocations + collocation_words = set() + for phrase, _ in all_candidates: + for word in phrase.split(): + collocation_words.add(word) + + for word in simple_keywords[:top_n]: + if word not in collocation_words: + all_candidates.append((word, 0.5)) + + # Remove duplicates + unique_candidates = {} + for phrase, score in all_candidates: + if phrase not in unique_candidates or score > unique_candidates[phrase]: + unique_candidates[phrase] = score + + sorted_candidates = sorted(unique_candidates.items(), + key=lambda x: x[1], reverse=True) + + # Deduplicate overlapping collocations + from .collocation_deduplicator import CollocationDeduplicator + deduplicated = CollocationDeduplicator.remove_overlapping_collocations( + sorted_candidates, 0.6 + ) + + return deduplicated[:top_n] \ No newline at end of file diff --git a/app/keyword_extraction/keyword_set.py b/app/keyword_extraction/keyword_set.py new file mode 100644 index 0000000..6fb0b73 --- /dev/null +++ b/app/keyword_extraction/keyword_set.py @@ -0,0 +1,383 @@ +""" +Main keyword set class with multilingual support and semantic deduplication. +""" + +import os +import re +import warnings +from collections import defaultdict +from typing import List, Tuple, Set, Dict, Optional + +from sentence_transformers import SentenceTransformer, util + +from .language_detection import detect_language_simple +from .paragraph_processor import ParagraphProcessor +from .collocation_deduplicator import CollocationDeduplicator +from .keyword_processor import KeywordProcessor +from .lemmatization import lemmatize_russian_collocation + +warnings.filterwarnings('ignore') + +class MultilingualKeywordSet: + """Multilingual keyword set with collocation deduplication.""" + + def __init__(self, + initial_keywords: Dict[str, List[str]] = None, + similarity_threshold: float = 0.75, + collocation_overlap_threshold: float = 0.6, + embedding_model: str = 'cointegrated/rubert-tiny2', + cache_folder: str = './model_cache', + use_collocations: bool = True, + max_ngram_size: int = 3, + max_chunk_size: int = 1000, + chunk_overlap: int = 100): + """ + Initialize multilingual keyword set. + """ + self.similarity_threshold = similarity_threshold + self.collocation_overlap_threshold = collocation_overlap_threshold + self.use_collocations = use_collocations + self.max_ngram_size = max_ngram_size + + self.paragraph_processor = ParagraphProcessor(max_chunk_size, chunk_overlap) + self.deduplicator = CollocationDeduplicator() + + if use_collocations: + print("Collocations enabled") + + # Create cache folder + os.makedirs(cache_folder, exist_ok=True) + + # Load model + print(f"Loading multilingual model from: {embedding_model}") + try: + self.model = SentenceTransformer(embedding_model, cache_folder=cache_folder) + print(f"Model loaded successfully") + except Exception as e: + print(f"Could not load {embedding_model}, falling back to English model...") + self.model = SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L3-v2', + cache_folder=cache_folder) + print(f"English model loaded as fallback") + + # Initialize processors + self.keyword_processor = KeywordProcessor(keybert_model=None) # Will be set after model load + self.keybert = None # Will be initialized later if needed + + # Try to initialize KeyBERT with the model + try: + from keybert import KeyBERT + self.keybert = KeyBERT(model=self.model) + self.keyword_processor.keybert = self.keybert + except Exception as e: + print(f"KeyBERT not available ({e}), using fallback extraction") + self.keybert = None + self.keyword_processor.keybert = None + + # Store keywords per language + self.keywords_by_lang: Dict[str, Set[str]] = {} + self.keyword_embeddings: Dict[str, Dict[str, any]] = {} + self.keyword_frequencies: Dict[str, Dict[str, int]] = {} + self.keyword_metadata: Dict[str, Dict] = {} + + # Add initial keywords + if initial_keywords: + for lang, keywords in initial_keywords.items(): + if lang not in self.keywords_by_lang: + self.keywords_by_lang[lang] = set() + self.keyword_embeddings[lang] = {} + self.keyword_frequencies[lang] = defaultdict(int) + + for keyword in keywords: + self.add_keyword(keyword, lang, check_similarity=False) + + # Public methods + + def add_keyword(self, keyword: str, language: str, check_similarity: bool = True, verbose = False) -> bool: + """Add a keyword to the set.""" + # Prepare keyword + keyword_lower = self.keyword_processor.prepare_keyword_for_addition(keyword, language) + + # Check if should skip + existing_keywords = self.keywords_by_lang.get(language, set()) + should_skip, reason = self.keyword_processor.should_skip_keyword( + keyword_lower, language, existing_keywords + ) + if should_skip: + if verbose: + print(f" Skipped '{keyword}' ({language}) - {reason}") + return False + + # Initialize language storage + if language not in self.keywords_by_lang: + self.keywords_by_lang[language] = set() + self.keyword_embeddings[language] = {} + self.keyword_frequencies[language] = defaultdict(int) + + # Check if already exists + if keyword_lower in self.keywords_by_lang[language]: + self.keyword_frequencies[language][keyword_lower] += 1 + return False + + # Check semantic similarity + if check_similarity and self.keywords_by_lang[language]: + try: + new_embedding = self.model.encode(keyword_lower, convert_to_tensor=True) + + max_similarity = 0 + most_similar_keyword = None + + for existing_keyword, existing_embedding in self.keyword_embeddings[language].items(): + similarity = util.pytorch_cos_sim(new_embedding, existing_embedding).item() + if similarity > max_similarity: + max_similarity = similarity + most_similar_keyword = existing_keyword + + if max_similarity >= self.similarity_threshold: + if verbose: + print(f" Rejected '{keyword}' -> '{keyword_lower}' (similar to '{most_similar_keyword}': {max_similarity:.3f})") + return False + else: + self._add_to_storage(keyword, keyword_lower, language, new_embedding) + if verbose: + print(f" Added: '{keyword}' -> '{keyword_lower}' (max similarity: {max_similarity:.3f})") + return True + except Exception as e: + if verbose: + print(f" Error adding keyword '{keyword}': {e}") + return False + else: + try: + embedding = self.model.encode(keyword_lower, convert_to_tensor=True) + self._add_to_storage(keyword, keyword_lower, language, embedding) + if verbose: + print(f" Added: '{keyword}' -> '{keyword_lower}'") + return True + except Exception as e: + if verbose: + print(f" Error adding keyword '{keyword}': {e}") + return False + + def process_text(self, text: str, top_n: int = -1, verbose: bool = False) -> Dict: + """Process text (sentence or paragraph).""" + # Calculate top_n if not specified + if top_n < 0: + marks = r'[,;.!?]+[\n]+' + marks_num = len(re.split(marks, text)) + top_n = marks_num if marks_num > 1 else int(text.count(' ') / 3) + + language = detect_language_simple(text) + + if verbose: + print(f"\n{'='*70}") + print(f"Processing Text (Language: {language.upper()})") + print(f"{'='*70}") + print(f"Text: {text[:150]}...") + + all_keywords = [] + all_new_keywords = [] + + # Process based on text length + if len(text) > self.paragraph_processor.max_chunk_size: + chunks = self.paragraph_processor.process_paragraph(text, language) + if verbose: + print(f"\nSplit into {len(chunks)} chunk(s)") + + for chunk_info in chunks: + keywords, new_keywords = self._process_chunk(chunk_info, language, top_n, verbose) + all_keywords.extend(keywords) + all_new_keywords.extend(new_keywords) + else: + extracted = self.keyword_processor.extract_keywords_from_text(text, language, top_n, verbose) + + if verbose and extracted: + print(f"\nExtracted keywords (after deduplication):") + for i, (candidate, score) in enumerate(extracted[:5], 1): + print(f" {i}. '{candidate}' (score: {score:.3f})") + + for candidate, score in extracted: + lemmatized = self.keyword_processor.lemmatize_if_necessary(candidate) + if self.add_keyword(candidate, language, check_similarity=True, verbose=verbose): + all_new_keywords.append(lemmatized) + all_keywords.append(lemmatized) + else: + similar = self.find_similar_keywords(lemmatized, 1) + if similar: + all_keywords.append(similar[0][0]) + + # Filter final keywords + filtered_keywords = self.keyword_processor.filter_stopwords(list(set(all_keywords))) + + if verbose: + print(f"\nAdded {len(all_new_keywords)} new keyword(s)") + print(f"Total keywords: {sum(len(kw) for kw in self.keywords_by_lang.values())}") + + return { + 'language': language, + 'keywords_added': all_new_keywords, + 'total_keywords': sum(len(kw) for kw in self.keywords_by_lang.values()), + 'keywords': filtered_keywords + } + + # Helper methods + + def _add_to_storage(self, original: str, normalized: str, language: str, embedding): + """Add keyword to storage.""" + self.keywords_by_lang[language].add(normalized) + self.keyword_embeddings[language][normalized] = embedding + self.keyword_frequencies[language][normalized] = 1 + self.keyword_metadata[normalized] = {'language': language, 'original': original} + + def _process_chunk(self, chunk_info: Dict, language: str, top_n: int, verbose: bool): + """Process a single chunk.""" + if verbose: + print(f"\n{'#'*40}") + print(f"Processing Chunk {chunk_info['chunk_id']}") + print(f"{'#'*40}") + + extracted = self.keyword_processor.extract_keywords_from_text( + chunk_info['text'], language, top_n + ) + + if verbose and extracted: + print(f"\nExtracted candidates:") + for i, (candidate, score) in enumerate(extracted[:5], 1): + print(f" {i}. '{candidate}' (score: {score:.3f})") + + all_keywords = [] + all_new_keywords = [] + + for candidate, score in extracted: + lemmatized = self.keyword_processor.lemmatize_if_necessary(candidate) + if self.add_keyword(candidate, language, check_similarity=True, verbose=verbose): + all_new_keywords.append(lemmatized) + all_keywords.append(lemmatized) + else: + similar = self.find_similar_keywords(lemmatized, 1) + if similar: + all_keywords.append(similar[0][0]) + + return all_keywords, all_new_keywords + + def get_all_keywords(self, language: Optional[str] = None, sort_by_frequency: bool = False) -> Dict[str, List[str]]: + """ + Get all keywords, optionally filtered by language. + """ + if language: + if language in self.keywords_by_lang: + keywords = list(self.keywords_by_lang[language]) + if sort_by_frequency: + keywords.sort(key=lambda x: self.keyword_frequencies[language].get(x, 0), reverse=True) + return {language: keywords} + else: + return {language: []} + else: + result = {} + for lang, keywords in self.keywords_by_lang.items(): + sorted_keywords = list(keywords) + if sort_by_frequency: + sorted_keywords.sort(key=lambda x: self.keyword_frequencies[lang].get(x, 0), reverse=True) + result[lang] = sorted_keywords + return result + + def find_similar_keywords(self, keyword: str, top_n: int = 5) -> List[Tuple[str, float, str]]: + """ + Find similar keywords across all languages. + + Returns: + List of (keyword, similarity_score, language) tuples + """ + keyword_lower = keyword.lower().strip() + + try: + keyword_embedding = self.model.encode(keyword_lower, convert_to_tensor=True) + + similarities = [] + for lang, embeddings in self.keyword_embeddings.items(): + for existing_keyword, existing_embedding in embeddings.items(): + similarity = util.pytorch_cos_sim(keyword_embedding, existing_embedding).item() + similarities.append((existing_keyword, similarity, lang)) + + similarities.sort(key=lambda x: x[1], reverse=True) + return similarities[:top_n] + except Exception as e: + print(f"Error finding similar keywords: {e}") + return [] + + def set_similarity_threshold(self, threshold: float): + """Adjust the similarity threshold.""" + self.similarity_threshold = threshold + print(f"Similarity threshold updated to {threshold}") + + def display_keyword_set(self, show_frequencies: bool = False): + """Display all keywords organized by language.""" + print(f"\nCurrent Multilingual Keyword Set") + print("=" * 60) + + for lang in sorted(self.keywords_by_lang.keys()): + count = len(self.keywords_by_lang[lang]) + print(f"\n{lang.upper()} ({count} keywords):") + print("-" * 40) + + if show_frequencies: + sorted_items = sorted(self.keyword_frequencies[lang].items(), + key=lambda x: x[1], reverse=True) + for i, (kw, freq) in enumerate(sorted_items[:20], 1): + print(f"{i:2d}. {kw:<35} (freq: {freq})") + else: + for i, kw in enumerate(sorted(self.keywords_by_lang[lang])[:20], 1): + print(f"{i:2d}. {kw}") + + if count > 20: + print(f" ... and {count - 20} more") + + print("=" * 60) + + def save_keywords_to_file(self, filename: str): + """Save current keywords to a file.""" + with open(filename, 'w', encoding='utf-8') as f: + f.write("# Multilingual Keyword Set\n") + f.write(f"# Similarity Threshold: {self.similarity_threshold}\n") + f.write(f"# Collocation Overlap Threshold: {self.collocation_overlap_threshold}\n\n") + + for lang in sorted(self.keywords_by_lang.keys()): + f.write(f"\n## {lang.upper()} Keywords:\n") + for kw in sorted(self.keywords_by_lang[lang]): + freq = self.keyword_frequencies[lang].get(kw, 1) + f.write(f"{kw}\t{freq}\t{lang}\n") + + print(f"Keywords saved to {filename}") + + def load_keywords_from_file(self, filename: str): + """Load keywords from a file.""" + if not os.path.exists(filename): + print(f"File {filename} not found") + return + + with open(filename, 'r', encoding='utf-8') as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + if not line or line.startswith('#') or line.startswith('##'): + continue + + parts = line.split('\t') + if len(parts) >= 2: + keyword = parts[0] + freq = int(parts[1]) if len(parts) > 1 else 1 + lang = parts[2] if len(parts) > 2 else 'en' + + if lang not in self.keywords_by_lang: + self.keywords_by_lang[lang] = set() + self.keyword_embeddings[lang] = {} + self.keyword_frequencies[lang] = defaultdict(int) + + if keyword not in self.keywords_by_lang[lang]: + self.keywords_by_lang[lang].add(keyword) + try: + self.keyword_embeddings[lang][keyword] = self.model.encode(keyword, convert_to_tensor=True) + self.keyword_frequencies[lang][keyword] = freq + except Exception as e: + print(f"Warning: Could not load '{keyword}': {e}") + + print(f"Loaded keywords from {filename}") \ No newline at end of file diff --git a/app/keyword_extraction/language_detection.py b/app/keyword_extraction/language_detection.py new file mode 100644 index 0000000..1eff936 --- /dev/null +++ b/app/keyword_extraction/language_detection.py @@ -0,0 +1,61 @@ +""" +Simple language detection for English and Russian. +No external libraries. +""" + +import re +from .stopwords import get_stopwords + +def detect_language_simple(text: str) -> str: + """ + Enhanced language detection using stopwords and character detection. + Returns 'en' for English, 'ru' for Russian. + """ + # Clean the text (remove punctuation, convert to lowercase) + clean_text = re.sub(r'[^\w\s]', ' ', text.lower()) + words = clean_text.split() + + if not words: + return 'en' + + # Method 1: Cyrillic character detection (быстрый, но неточный) + cyrillic_range = r'[а-яА-ЯёЁ]' + cyrillic_chars = len(re.findall(cyrillic_range, text)) + + # Method 2: Stopword-based detection (более точный) + ru_stopwords = get_stopwords('ru') + en_stopwords = get_stopwords('en') + + # Count stopwords from each language + ru_stopword_count = sum(1 for word in words if word in ru_stopwords) + en_stopword_count = sum(1 for word in words if word in en_stopwords) + + # Method 3: Common Russian indicators (для коротких текстов) + russian_indicators = [ + 'это', 'что', 'который', 'также', 'ещё', 'очень', 'можно', + 'нельзя', 'нужно', 'почему', 'потому', 'поэтому', 'итак' + ] + ru_indicator_count = sum(1 for word in words if word in russian_indicators) + + # Decision logic + # If strong Cyrillic presence -> Russian + if cyrillic_chars > len(text) * 0.3: + return 'ru' + + # If strong stopword evidence + if ru_stopword_count > en_stopword_count * 1.5: + return 'ru' + + if en_stopword_count > ru_stopword_count * 1.5: + return 'en' + + # If many Russian function words + if ru_indicator_count >= 2: + return 'ru' + + # Check for Russian-specific patterns + if any(word.endswith(('ться', 'тся', 'щий', 'щая', 'щее')) for word in words): + return 'ru' + + # Default to English + return 'en' \ No newline at end of file diff --git a/app/keyword_extraction/lemmatization.py b/app/keyword_extraction/lemmatization.py new file mode 100644 index 0000000..aba04da --- /dev/null +++ b/app/keyword_extraction/lemmatization.py @@ -0,0 +1,62 @@ +""" +Words lemmatization utilities for Russian. +Lemmatizes words in Russian. +""" +import pymorphy3 + +morph_ru = pymorphy3.MorphAnalyzer() + +def lemmatize_russian(word: str) -> str: + """ + Приводит русское слово к начальной форме. + Если лемматизация не удалась, возвращает исходное слово. + """ + try: + parsed = morph_ru.parse(word)[0] + return parsed.normal_form + except: + return word + +def lemmatize_russian_collocation(phrase: str) -> str: + """ + Лемматизирует всю коллокацию (фразу из нескольких слов). + Пример: "глубокие нейронные сети" -> "глубокий нейронный сеть" + """ + if not phrase or len(phrase.strip()) == 0: + return phrase + + words = phrase.lower().split() + lemmatized_words = [lemmatize_russian(word) for word in words] + return ' '.join(lemmatized_words) + +def is_function_word(word: str) -> bool: + """ + Определяет, является ли слово служебной частью речи. + Возвращает True для слов, которые стоит исключить из ключевых слов. + """ + try: + parsed = morph_ru.parse(word)[0] + pos = parsed.tag.POS + + # Служебные части речи (стоит исключить) + function_positions = {'PREP', 'CONJ', 'PRCL', 'INTJ', 'NPRO'} + + # Также исключаем числительные + if pos == 'NUMR': + return True + + return pos in function_positions + except: + return False + +def is_content_word(word: str) -> bool: + """ + Определяет, является ли слово знаменательной частью речи. + """ + try: + parsed = morph_ru.parse(word)[0] + pos = parsed.tag.POS + content_positions = {'NOUN', 'ADJF', 'ADJS', 'VERB', 'INFN'} + return pos in content_positions + except: + return False \ No newline at end of file diff --git a/app/keyword_extraction/paragraph_processor.py b/app/keyword_extraction/paragraph_processor.py new file mode 100644 index 0000000..db65006 --- /dev/null +++ b/app/keyword_extraction/paragraph_processor.py @@ -0,0 +1,104 @@ +""" +Paragraph processing utilities for splitting long texts into chunks. +""" + +import re +from typing import List, Dict + +class ParagraphProcessor: + """Handles processing of paragraphs into chunks.""" + + def __init__(self, max_chunk_size: int = 500, overlap: int = 50): + """ + Initialize paragraph processor. + + Args: + max_chunk_size: Maximum characters per chunk + overlap: Number of characters to overlap between chunks + """ + self.max_chunk_size = max_chunk_size + self.overlap = overlap + + def split_into_sentences(self, text: str, language: str = 'en') -> List[str]: + """Split text into sentences.""" + sentence_endings = r'[.!?]+[\s\n]+' + sentences = re.split(sentence_endings, text) + return [s.strip() for s in sentences if s.strip()] + + def split_paragraph_into_chunks(self, paragraph: str, language: str = 'en') -> List[Dict]: + """ + Split a paragraph into overlapping chunks. + + Returns: + List of dicts with 'chunk_id', 'text', 'start_sentence', 'end_sentence' + """ + sentences = self.split_into_sentences(paragraph, language) + + if not sentences: + return [] + + chunks = [] + current_chunk = [] + current_length = 0 + chunk_id = 1 + + for i, sentence in enumerate(sentences): + sentence_length = len(sentence) + + if current_length + sentence_length > self.max_chunk_size and current_chunk: + chunk_text = ' '.join(current_chunk) + chunks.append({ + 'chunk_id': chunk_id, + 'text': chunk_text, + 'start_sentence': chunk_id - 1, + 'end_sentence': i + }) + chunk_id += 1 + + # Start new chunk with overlap + overlap_sentences = [] + overlap_length = 0 + for s in reversed(current_chunk): + if overlap_length + len(s) <= self.overlap: + overlap_sentences.insert(0, s) + overlap_length += len(s) + else: + break + + current_chunk = overlap_sentences + current_length = overlap_length + + current_chunk.append(sentence) + current_length += sentence_length + 1 + + # Add the last chunk + if current_chunk: + chunk_text = ' '.join(current_chunk) + chunks.append({ + 'chunk_id': chunk_id, + 'text': chunk_text, + 'start_sentence': chunk_id - 1, + 'end_sentence': len(sentences) + }) + + return chunks + + def process_paragraph(self, paragraph: str, language: str = 'en') -> List[Dict]: + """ + Process a paragraph into chunks ready for keyword extraction. + """ + # Remove extra whitespace + paragraph = re.sub(r'\s+', ' ', paragraph).strip() + + # Check if paragraph needs to be split + if len(paragraph) <= self.max_chunk_size: + return [{ + 'chunk_id': 1, + 'text': paragraph, + 'is_full_paragraph': True + }] + + # Split into chunks + chunks = self.split_paragraph_into_chunks(paragraph, language) + + return chunks \ No newline at end of file diff --git a/app/keyword_extraction/requirements.txt b/app/keyword_extraction/requirements.txt new file mode 100644 index 0000000..489546f --- /dev/null +++ b/app/keyword_extraction/requirements.txt @@ -0,0 +1,5 @@ +sentence-transformers +keybert +numpy +nltk +pymorphy3 \ No newline at end of file diff --git a/app/keyword_extraction/stopwords.py b/app/keyword_extraction/stopwords.py new file mode 100644 index 0000000..011b5ff --- /dev/null +++ b/app/keyword_extraction/stopwords.py @@ -0,0 +1,128 @@ +""" +Language-specific stopwords for English and Russian. +""" + +STOPWORDS = { + 'en': { + # Предлоги + 'a', 'an', 'the', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', + 'through', 'across', 'along', 'around', 'behind', 'beneath', 'beside', + 'between', 'beyond', 'during', 'except', 'from', 'into', 'like', 'near', + 'off', 'onto', 'over', 'past', 'since', 'under', 'upon', 'within', 'without', + + # Союзы + 'and', 'or', 'but', 'so', 'for', 'nor', 'yet', 'because', 'if', 'then', + 'else', 'when', 'where', 'which', 'while', 'than', 'that', 'though', + 'although', 'whereas', 'whether', 'either', 'neither', 'both', + + # Местоимения + 'i', 'you', 'he', 'she', 'it', 'we', 'they', + 'me', 'him', 'her', 'us', 'them', + 'my', 'your', 'his', 'her', 'its', 'our', 'their', + 'mine', 'yours', 'hers', 'ours', 'theirs', + 'this', 'that', 'these', 'those', + 'what', 'which', 'who', 'whom', 'whose', + 'anyone', 'everyone', 'someone', + 'no one', 'nobody', + + # Частицы и модальные слова + 'not', 'no', 'never', 'just', 'even', 'only', 'very', 'so', 'too', 'also', + 'either', 'neither', 'both', 'all', 'some', 'any', 'no', 'every', 'each', + 'few', 'many', 'much', 'most', 'least', 'little', 'lot', 'lots', + + # Вспомогательные и модальные глаголы + 'be', 'am', 'are', 'is', 'was', 'were', 'been', 'being', 'have', 'has', + 'had', 'having', 'do', 'does', 'did', 'doing', 'can', 'could', 'will', + 'would', 'shall', 'should', 'may', 'might', 'must', 'ought', 'need', + + # Вопросительные слова + 'how', 'why', 'where', 'when', 'what', 'which', 'who', 'whom', 'whose', + + # Числительные + 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', + 'first', 'second', 'third', 'fourth', 'fifth', 'last', 'next', 'previous', + + # Короткие слова (обычно не несут смысла) + 'as', 'at', 'by', 'in', 'of', 'on', 'to', 'up', 'down', 'out', 'off', + 'over', 'under', 'again', 'further', 'once', 'here', 'there', 'now', 'then', + 'than', 'via', 'per', + }, + 'ru': { + # Предлоги + 'в', 'во', 'на', 'за', 'под', 'над', 'перед', 'после', 'через', 'сквозь', + 'между', 'среди', 'около', 'возле', 'мимо', 'вдоль', 'против', 'ради', + 'для', 'без', 'вместо', 'из', 'из-за', 'из-под', 'благодаря', 'согласно', + 'вопреки', 'наперекор', 'вследствие', 'ввиду', 'вроде', 'наподобие', + 'относительно', 'посредством', 'путём', 'через', 'сквозь', 'по', 'к', + 'у', 'о', 'об', 'обо', 'при', 'про', 'через', + + # Союзы + 'и', 'а', 'но', 'да', 'или', 'либо', 'то', 'если', 'же', 'ведь', 'однако', + 'зато', 'чтобы', 'потому', 'так', 'как', 'будто', 'словно', 'точно', 'чем', + 'нежели', 'хотя', 'пусть', 'пускай', 'лишь', 'только', 'также', 'тоже', + 'причём', 'притом', 'зачем', 'отчего', 'почему', 'потому', 'оттого', + + # Местоимения + 'я', 'ты', 'он', 'она', 'оно', 'мы', 'вы', 'они', 'меня', 'тебя', 'его', + 'ее', 'нас', 'вас', 'их', 'мне', 'тебе', 'ему', 'ей', 'нам', 'вам', 'им', + 'мой', 'твой', 'его', 'ее', 'наш', 'ваш', 'их', 'свой', 'этот', 'тот', + 'такой', 'таков', 'столько', 'сколько', 'несколько', 'весь', 'всякий', + 'каждый', 'любой', 'другой', 'иной', 'сам', 'самый', 'кто', 'что', + 'это', 'то', 'та', + + # Частицы + 'не', 'ни', 'бы', 'б', 'же', 'ж', 'ли', 'ль', 'пусть', 'пускай', 'да', + 'давай', 'ага', 'ой', 'ах', 'увы', 'вот', 'вон', 'разве', 'неужели', + 'едва', 'чуть', 'почти', 'прямо', 'точно', 'ровно', 'именно', + + # Вспомогательные и модальные глаголы + 'быть', 'стать', 'являться', 'становиться', 'бывать', 'бывал', 'бывали', 'бывало', 'будет', + 'будут', 'был', 'была', 'было', 'были', 'есть', 'суть', 'мочь', 'смочь', 'уметь', + 'хотеть', 'захотеть', 'желать', 'должен', 'должна', 'должно', 'должны', + 'обязан', 'годиться', 'пригодиться', 'следовать', 'полагаться', + + # Вопросительные слова + 'кто', 'некто', 'никто', 'что', 'нечто', 'ничто', 'какой', 'никакой', 'какая', + 'никакая', 'некая', 'какое', 'никакое', 'некое', 'какие', 'некие', 'никакие', + 'чей', 'чья', 'чьё', 'чьи', + 'который', 'некоторый', 'которая', 'некоторая', 'которое', 'некоторое', 'которые', 'некоторые', + 'где', 'нигде', 'куда', 'никуда', 'некуда', 'откуда', 'неоткуда', 'ниоткуда', + 'когда', 'некогда', 'почему', 'зачем', 'незачем', 'отчего', 'ниотчего', 'неотчего', + 'сколько', 'несколько', 'нисколько', 'насколько', 'нинасколько', + 'что', 'чего', 'чему', 'чем', 'чём', + 'то', 'того', 'тому', 'тем', 'том', + 'кто', 'кого', 'кому', 'кем', 'ком', + 'тот', 'того', 'тому', 'тем', + + # Числительные + 'один', 'одна', 'одно', 'одни', 'два', 'две', 'три', 'четыре', 'пять', + 'шесть', 'семь', 'восемь', 'девять', 'десять', + 'первый', 'второй', 'третий', 'четвёртый', 'пятый', + 'последний', 'предыдущий', 'следующий', + + # Короткие слова + 'из', 'со', 'об', 'обо', 'от', 'до', 'по', 'без', 'для', 'про', 'у', 'за', + 'над', 'под', 'перед', 'после', 'через', 'между', 'сквозь', 'вслед', + + # Междометия + 'о', 'ох', 'ах', 'эх', 'ух', 'ну', 'браво', 'ура', 'ого', 'ба', + } +} + +PUNCTUATION = { + 'en': r'[^\w\s]', + 'ru': r'[^\w\s\u0400-\u04FF]' # Preserve Cyrillic characters +} + +def get_stopwords(language: str = 'en'): + """Get stopwords for the specified language.""" + return STOPWORDS.get(language, STOPWORDS['en']) + +def get_punctuation_pattern(language: str = 'en'): + """Get punctuation pattern for the specified language.""" + return PUNCTUATION.get(language, PUNCTUATION['en']) + +# Дополнительная функция для проверки, является ли слово стоп-словом +def is_stopword(word: str, language: str = 'en') -> bool: + """Check if a word is a stopword.""" + return word.lower() in get_stopwords(language) \ No newline at end of file diff --git a/app/main.py b/app/main.py index 9da30ae..042a32c 100644 --- a/app/main.py +++ b/app/main.py @@ -29,6 +29,11 @@ def root(): def health(): return {"status": "healthy", "database_url": os.getenv("DATABASE_URL", "not set")} +@app.post("/search") +async def search(query: str, top_k: int = 10): + from app.search import search_by_query + return await search_by_query(query, top_k) + """ import docs_parser from fastapi import FastAPI, File, UploadFile diff --git a/app/qdrant_client.py b/app/qdrant_client.py new file mode 100644 index 0000000..9589c44 --- /dev/null +++ b/app/qdrant_client.py @@ -0,0 +1,147 @@ +from qdrant_client import QdrantClient +from qdrant_client.http import models +from typing import List, Dict +import logging + +logger = logging.getLogger(__name__) + +# Подключение к Qdrant +qdrant = QdrantClient(host="localhost", port=6333) + +COLLECTION_NAME = "chunks" +VECTOR_SIZE = 312 + +def ensure_collection(): + """Создаёт коллекцию в Qdrant, если её нет""" + try: + collections = qdrant.get_collections().collections + if not any(c.name == COLLECTION_NAME for c in collections): + qdrant.create_collection( + collection_name=COLLECTION_NAME, + vectors_config=models.VectorParams( + size=VECTOR_SIZE, + distance=models.Distance.COSINE + ) + ) + logger.info(f"Collection '{COLLECTION_NAME}' created") + else: + logger.debug(f"Collection '{COLLECTION_NAME}' already exists") + except Exception as e: + logger.error(f"Failed to ensure collection: {e}") + +def index_chunk( + chunk_id: str, + document_id: str, + chunk_index: int, + text: str, + keywords: List[str], + language: str, + embedding: List[float] +) -> bool: + """Сохраняет вектор и метаданные чанка в Qdrant""" + ensure_collection() + try: + qdrant.upsert( + collection_name=COLLECTION_NAME, + points=[ + models.PointStruct( + id=chunk_id, + vector=embedding, + payload={ + "document_id": document_id, + "chunk_index": chunk_index, + "text": text, + "keywords": keywords, + "language": language + } + ) + ] + ) + logger.debug(f"Indexed chunk {chunk_id}") + return True + except Exception as e: + logger.error(f"Failed to index chunk {chunk_id}: {e}") + return False + +def semantic_search(query_embedding: List[float], top_k: int = 10) -> List[Dict]: + """Семантический поиск по вектору (KNN)""" + ensure_collection() + try: + # Используем query_points вместо search (для новых версий Qdrant) + results = qdrant.query_points( + collection_name=COLLECTION_NAME, + query=query_embedding, + limit=top_k, + with_payload=True + ) + return [ + { + "id": hit.id, + "score": hit.score, + **hit.payload + } + for hit in results.points + ] + except AttributeError: + # Для старых версий Qdrant (fallback) + try: + results = qdrant.search( + collection_name=COLLECTION_NAME, + query_vector=query_embedding, + limit=top_k, + with_payload=True + ) + return [ + { + "id": hit.id, + "score": hit.score, + **hit.payload + } + for hit in results + ] + except Exception as e2: + logger.error(f"Both search methods failed: {e2}") + return [] + except Exception as e: + logger.error(f"Semantic search failed: {e}") + return [] + +def delete_chunks_by_document(document_id: str) -> bool: + """Удаляет все чанки документа из Qdrant""" + try: + qdrant.delete( + collection_name=COLLECTION_NAME, + points_selector=models.FilterSelector( + filter=models.Filter( + must=[ + models.FieldCondition( + key="document_id", + match=models.MatchValue(value=document_id) + ) + ] + ) + ) + ) + logger.info(f"Deleted chunks for document {document_id}") + return True + except Exception as e: + logger.error(f"Failed to delete chunks for document {document_id}: {e}") + return False + +def delete_all_chunks(): + """Очищает всю коллекцию (для тестов)""" + try: + qdrant.delete_collection(COLLECTION_NAME) + logger.info("Collection deleted") + except Exception as e: + logger.error(f"Failed to delete collection: {e}") + +def get_chunk_count() -> int: + """Возвращает количество точек в коллекции""" + try: + ensure_collection() + info = qdrant.get_collection(COLLECTION_NAME) + return info.points_count + except Exception as e: + logger.error(f"Failed to get chunk count: {e}") + return 0 \ No newline at end of file diff --git a/app/search.py b/app/search.py new file mode 100644 index 0000000..e0eabf9 --- /dev/null +++ b/app/search.py @@ -0,0 +1,8 @@ +from app.qdrant_client import semantic_search, hybrid_search, delete_chunks_by_document +from app.embeddings import get_embedding + +async def search_by_query(query: str, top_k: int = 10): + query_embedding = get_embedding(query) + results = semantic_search(query_embedding, top_k) + # results уже содержат payload с document_id, text, keywords и т.д. + return results \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8d14a3c..7eba6eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,64 +1,40 @@ services: postgres: - image: postgres:15 + image: pgvector/pgvector:pg15 # используем готовый образ с pgvector (опционально, но можно оставить обычный postgres, если pgvector не нужен) container_name: docs_postgres - env_file: - - .env environment: - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} - POSTGRES_DB: ${POSTGRES_DB:-docs_db} + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: docs_db ports: - "5432:5432" volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-docs_db}"] + test: ["CMD-SHELL", "pg_isready -U postgres -d docs_db"] interval: 5s timeout: 5s retries: 5 networks: - docs_network - solr: - image: solr:9.1 - container_name: docs_solr + qdrant: + image: qdrant/qdrant:latest + container_name: docs_qdrant ports: - - "8983:8983" + - "6333:6333" # REST API + - "6334:6334" # gRPC API (опционально) volumes: - - solr_data:/var/solr - command: - - bash - - -c - - | - precreate-core chunks - cp -r /opt/solr/server/solr/configsets/_default/conf/* /var/solr/data/chunks/conf/ - curl -X POST -H 'Content-type:application/json' --data-binary '{"add-field":{"name":"embedding","type":"dense_vector","indexed":true,"stored":true,"dimensions":312}}' http://localhost:8983/solr/chunks/schema + - qdrant_storage:/qdrant/storage networks: - docs_network - # backend - временно закомментирован для локальной разработки - # backend: - # build: . - # container_name: docs_backend - # ports: - # - "8000:8000" - # depends_on: - # postgres: - # condition: service_healthy - # solr: - # condition: service_started - # env_file: - # - .env - # environment: - # DATABASE_URL: postgresql+asyncpg://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@postgres:5432/${DB_NAME:-docs_db} - # SOLR_URL: http://solr:8983/solr/chunks - # networks: - # - docs_network + # solr временно отключаем (можно закомментировать или удалить) + # solr: ... volumes: postgres_data: - solr_data: + qdrant_storage: networks: docs_network: diff --git a/requirements.txt b/requirements.txt index 5420ced..aa90c9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ sentence-transformers nltk pymorphy3 numpy -requests \ No newline at end of file +requests +qdrant-client \ No newline at end of file diff --git a/test_qdrant.py b/test_qdrant.py new file mode 100644 index 0000000..25b248e --- /dev/null +++ b/test_qdrant.py @@ -0,0 +1,43 @@ +import asyncio +import uuid +from app.embeddings import get_embedding +from app.qdrant_client import index_chunk, semantic_search, get_chunk_count, delete_all_chunks + +async def test(): + print("=== Testing Qdrant ===\n") + + # Очистка + print("1. Cleaning up...") + delete_all_chunks() + print(f" Chunks after cleanup: {get_chunk_count()}\n") + + # Индексация тестового чанка + print("2. Indexing test chunk...") + chunk_id = str(uuid.uuid4()) + text = "Нейронные сети и машинное обучение" + emb = get_embedding(text) + success = index_chunk( + chunk_id=chunk_id, + document_id="test-doc-123", + chunk_index=0, + text=text, + keywords=["нейронные сети", "машинное обучение"], + language="ru", + embedding=emb + ) + print(f" Indexing success: {success}") + print(f" Chunks after index: {get_chunk_count()}\n") + + # Семантический поиск + print("3. Semantic search...") + query = "глубокое обучение" + query_emb = get_embedding(query) + results = semantic_search(query_emb, top_k=3) + print(f" Found {len(results)} results") + for r in results: + print(f" - Score: {r.get('score', 0):.4f} | Text: {r.get('text', '')[:50]}...") + + print("\n✓ Test completed") + +if __name__ == "__main__": + asyncio.run(test()) \ No newline at end of file From a255c12cbf7e3726fbf42a4f778d9c51505cda97 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sat, 13 Jun 2026 00:11:02 +0400 Subject: [PATCH 16/34] Finish 1#2 --- app/__pycache__/auth.cpython-314.pyc | Bin 0 -> 14087 bytes app/__pycache__/main.cpython-314.pyc | Bin 2199 -> 3985 bytes app/__pycache__/schemas.cpython-314.pyc | Bin 0 -> 6201 bytes app/auth.py | 283 ++++++++++++++++++++++++ app/main.py | 219 ++++++------------ app/schemas.py | 88 +++++++- app/search.py | 2 +- 7 files changed, 440 insertions(+), 152 deletions(-) create mode 100644 app/__pycache__/auth.cpython-314.pyc create mode 100644 app/__pycache__/schemas.cpython-314.pyc create mode 100644 app/auth.py diff --git a/app/__pycache__/auth.cpython-314.pyc b/app/__pycache__/auth.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f62eab06cd5ae49298e02db1cc59277ea6ac9557 GIT binary patch literal 14087 zcmeHNYj7Lab>0OQ4+7v5e1PwjNRa}?haRLJv?YZiDS;M6K`Tm%64EF{tVqZp0Pij> zk!~AMcA|&f&~lP7ov5`u(M+XIW+FRQBge99IqszWw=kF?yy9VCN=9-V`&ZAo zi^T$bNKWnkXs0`adk^kC_qBV@cg{WcZpg{DQSe;g8&6$aPf`Dlf3(M(B<`MNC@MyA zltevBadZzY(IhuW2FMLPMvqZ4dKiiEm?Tp*@)&!}9*bo0SS70`OUm-tBpWGbdh8yD zy}75rKA5tsAU^(N`NWm&?;QQ~lQ8w?#fzu&l-URr_~xg&McyUX`mT zjjLg>rat*fh6^dGf*M@oJ*K@QJ?DzG`Bqb0iBvOK`-NJ$(pr^DwOm;V#g&&(p**RP zs{p!^&`n$w(A9))=2)QD5W0n13v>;kH*&Q=uOsxPK?i)jI<9`3_Ki0WX2Dyxc~9yy z)+la0=jIw(Xl}zcqgp1l4qAbGvo<&KuSEN|#lSVfNKH6Wh?d$&ZCk3A=9;0lC0V<5 z&;m2r$Zcw-xy{=&8Q3=qdfLg+HI%z!>?-tO^GBmC{xNB!IXce%TkQL+{9OF0_*wa;{3fKg;!n%B z%%Jj<_4y__yPK6F(0;G5Ia|Rr!YeCbYZB z%KrpSvFHux@byNU{Bx*!71$=B`nLR5d=dv?_xeSt!`s6GiQMW0;xSg8n61xkRICBN z#7n_ZUdh5Vz=tHiVnuQy66W2Ek_8=bkuixE6iXK$<--9{$@cX3d%MpJ@lh!l2`gq% z@=If)+n^L4?0_Y2_WDKfbVLa3;r#-%E6ptB_{Yb1QQ8|3MxlunmfkH05kWDYIxV?r z#nB;-hldWsn}ELIzggr%{1Dcdhk3~tIB}g;m?I)Dz~r2mbNp~ng!v`=RkHgdkMUuS z7o(A|$SY1xeVfTrGPdGS8zxzDU=F^+-JM)_zi)r{aitjlbKQHn?!%t+@@y>c=-qda z>*@C#P)we~o7=W3rKGg8v-|L2U-#GHZFTz&^z?*;wr*5P(mPAmcXb>;tkBz( zoQ^JF^&L6f&GmI0=vFAL#M^QB@Ueqj*F-0_)~CjXBN8j}5*rM&eE5l=5DAa+VQCF} zL=1+9p;Y9D#sr=X@K3NO{h`na|IlO26Zp3YY6M?K;w+RN8D1Z(WKZZW?1N$fWl)GB>^SpBnX>d5^=n>N_)#lWR!0SLlPcB84;fF zhXZ4yEu-UJA#w_YO>7B7hD4aVUl!@DV_3d_Dnu5o$a9#4<=?)Tb0f z?_R{@s&N6cpBTx%yHHD!1tWn^a>2~qY50Q+_G+(!RI3R`YoHya2a?cCQC@7jnrKyx#GmOU{fj7Oy= z6`#`ZC_dHvxoJ!~*|gnlQq1^%1H58ZxfRPW9~S)~NimQ3#gXV~#T?*~!EF?3U?3$& z`+~j`;}S0_3^I!-7_@fFfSv`?AlI+Hc zHISgtnP~GOCRaNbFuPqtvUh7qp}Mq`^gxs3(wMs;(q304Z#oy$;j;#1)1=brsV*l+ z>HF7f?i`UP5A-3wjFJTA3T+wMG?XW>PG{v`#AESi;%DQ}$gii1)U#}SGE?Bd>fLS@ zHb7?}I!%x%mZ1@T=&|T&p%DuWLK|cP3WSJ@s;g6_C-(_n2%a3*UwS)Kl6TnxM2z}K zm6kbc@vOBnZmpaiPFU;atR=J7s<^dk`gFotf7fJmm=`Fc$-G=>80dru%@>5R)cZyl zyKYnkg#nA~C(>};pfG{pkVItSe$ucX#ucl`3O=Q*^w;P`l3orhko>Bs{?{iAtmH|F z0wVEhFr@iq4+fzD?2kNNt48U$24(|xt!%)dQ5>y#!ub8PCu#uU%GLhqIcGR(;EWv8 zV$k}mrUvq~dT)V7={bloJpj7{^%*GED9)t$vLK%3v2R1UO@5i+-D`k$mryc)0kVnS z6UZl2{38l2cv2T2f1OobwkD{13u?i+Yqs_G_p&?Dq4RaDF^)+?0!pJ&wBke*^H>y}KLI}iQ5(gkTYm8SG(QIfb0>CV ziTcMw5q|(+{Y*3n4r&H4JHfj^qo~f%C!r4}9D;U4IQxN!Q42OI*Etou5Pasb*uMGf zqN#z41DDso)c9g!BD?-4?wN-V%`|Yo&gNo!=NS9BzRAAlx~I2Z*?Mie>};K3+UD&! zQ|%YqpBuhB@!Ihj`)--pt&6l~Kfja?3W)Xs?#J(^ZbzX$P2D_1-8}SV3DwO*qtVA) zpBwO#=l|6$Q{5)vtt?Zw*HA6wpDj~0&7oXb*EDZE&B0z>)3&klYoK(nby8IgAa`68 z_P}g}PDm6>ax9@6{<$5(KCD6Y63~7T^%8oJBI@N_VQc}VK25UP@#nmG;TZK3!bD z5>?X7nMp;0TA22A)w>bU;NDNa{7|3rsI9aU)ya8TMD223mB>GRvcOB8lx{r>e`C~x zy_x|;3i9Y?3xXiDW)Pw@7@w6q>b)5}+|EAsQ;7rjE9@JXOF#|aD#**=5A1a;6I$~9K1 zUA_qe-i81G{)G$!1My4XXq^YH+pMlj^37%&m=#;ftcXMvgf5u+=Xu9qpE=BuFUg`I za+_7zMn6aB#RP>}F@ke0pmm2Y{Y@-A2#M-|CEa(Y>hNb;i6P)3-yFTM7IcGuG(&A-FR{)r!%&1&XRNP$;l_5+nTVHO^?i2>gIEbQ9*eg|d1T7WC)I>Po=;KQFApqxg9ct_6fd;V&S` z5KD{!s~C&2MZ{7#0aS)O7AEB}6Kqcc8~Lyg0g)k)GwkMfvu=Sz?q4F2!5K^a$`UzY zC*ii1P-HlWAqgu{qa#9SqN+1877CD%&J+Gn5S=j=z&`*1ouEG?g4!KB)`S5yEEs}X z#jHlut{WA*8guih`|AKN`GX>LbUBXBQy^lfX@c+_~(LZCaf$`QJp>R%D`b$HkJ zEZSHr?ND?sZ)#tnPo>|u9UKX*y_WBTQ>s%i=QDK7*lV&;QKp~qI`vi>mD-P)`c2DM z&~K`tSV}i(W-0)Cu?l!s`qvzxgVaODN-9WOsNS|VV~}E1R?3{MJovMu2_`q>*TJfiT^U3I$4>OQZNd;NQ$R#eihU69?dtC9@9F5p z$Zco0x4-9LA3+Mm!k>XiD(sGOkRJtGF`XXa1zxd^LBKHVhlqKBIOGrc1s?>;eG$PI z#(=q5HGC0lh1O3P!=@0B6vrU$#b8?zrMM0vT3Spi4`W&owiQ6neaIo;c?40G7lHUa zNclpZ*|cuHVBHlyQP6naGUuv($^N4KN_WE5aNas+cfMyYpD$Q1JJv54DMtnDvkHq} zXus5c`Q+99ABASCHpZ(qzE=42%2z7iULW7|NTO~*oO zIg9=5U(V;$%GTOBdv0w1pYG*Sj>3Bs4TF=?1tXB3FPf>G1N3L&dYIi=dQUC=blsk8 z>K{4`K;CJj_pD{^G_gRxlU=Z<+W1a219S!^p~j(h{x{>jYM{iC@ycqUI{|g=egO0% zs23|BK>%{lu0>4d3yK$vkbh*#S~Nkf`w_rM9*loIg|LH$L4_TNim7!ODMu5J5Jmux z5En=uUGcp5>6bhlgfg^aFz^_u8j2_+Gj<*rGZhj&JXEur;6F~GLc#&jIaC#Nx(zIcLK?lQGu@vAHbUf|YW(WTxaZ z5k>86%O-llM9i?l;GrCxJDHKD2k`u$Ic6xp^wfn4wkf*m&uhA&&ErPp|@zx1{wZ^6A zdXLLn46quXw??D%oUS5qrr4HCdZY+InF|ObunBy6Ij!I4T_c9wOLi z8~Xz^w8>Wh^nRAy)Z*b3IZQ{xk0ANheI1l$TN5k)An9#r79GwRNK-gn2frcd@=UUs zhqT~*EIpi9y^(f~i@+~}>*~pbrqOAEknl|c)CBF3yB;S9 z9DNhTN0QD5>kmP^H89Rfehjq!HM|eu3?_dC2^@-`+dgk*^(+^8pq+W9uif z>)F!~KV}iE*?NVpcRK|UTctc6LBhYJrzeoo5xp>p=o@`|I=bMb{Lqo^!~FuDK#+qx zfkkQyB#O}=fPnE(WOOtrk&T*ShKtk#2GdXjPyr}~h4qSe8#<3xt{xG-rJoAw~cP_u^ zh005n^Dg(b+MhOEZ%Vkf&t>P|voH>GtP7knljYpr$-U=WpYENrIi}nf+|O*7Wr}2` z2>&)*Xn3waVJn+u%4DYOBgS@a|KxtTpyo=~-yeASz?I!Mn%-u9m30SppZ~H(&g-3F z4iFdZ4{cNqM=v%~rsDURvd=^e)t~KfbhGrG9J;%dxs$sK=y!5D9)j%MEV{dbdDrGb zdM({u#k^ay4d{QapfRn&bi%A{RfX?BUtgW!1F2jK9V!eS#%qQT@AW;<@cjmuQiiV> zPdg!*x792dAt#0p^5upPoh@ScZh?sWUod>QXXU|WfpeKMeqb_E<`34Je2yAdGKJbv ztY-Fx0q&tMr`Ae8A@de%WqOW=6wcge{mcyRWtd^470i4K9c2bmW^Ev4sB~j8kTOkr z8E4TPVTAqO3k2+~V^NrcP5u!M~cdpd`2(?u`aCrgtWJ8iZRzye*CXB$WgdirTFnJsjIIwl3pMNRY8F3|d z?gIc8PSLmY|j`<*sMOxK_HRp}9;X?v-*kp{L}N8TU%* zzk}#Mjz}IZb+nO;gym{AdOnb^(Yj--jqWYd>hxUaS_xBDjf7b;HUt<6OQDJ0x8};8 zR5t{v!te0^%t(Tgm5jPg! zD@5GB=!^hYITIjj%DvBs&`lB?w2#GLB{uo*@B$_nm{UXbD3m08AWS1AVzR1dx&#Q#Y5wFvY)E`q;#p!qZCL;gHv9xA zRPYytzXhgy$oQX-sDNlQ+2#xCu1K@>JL2^_5(Vu`gK!_@mdqEFO+TE_!f<)6g~HWg zbZHna!y9eL1j?E;Z`6hW>5}u;J^&p5bZ-rb+?inB$smb}x(zF!t`WFCU(8d3dTvbnY_#au);igt29m&%jrVb->P-x{KSp z2VLcC!ShiAaSKU2=z#&eQt;#m;9~&KfB`&6F2k8HWXt1rI2Qt?C^%Z&E*W2=$1r5I zvWtLwzc=Bc_A{^%ON*=}L*8(84fPFATeYw^aufGt^@w=$bxN3o5z|9nOz1VZHE;~{?tuGWo?H}hwtWdU5XM)u1NG#xXgIEM`Hy0MA4taG&@RmZQ%^a6*MF{DQ(2 zOi+E2d)0?w;bS3wmv9rx;PX;q8%QLC&}n+nV5FJ5R*JU$hRXSX+8n1ge@yL6P&+@M zdOx75enVydmTLJeRrw!O^>3;2PYov8a2M`&8WwUWTmEcT*?U=K)7ukSwJ|feOgVW| z$1fh2OX?HZ>tnXLqLLSmKYv`V-j*oZE>i__j-08Ui#^XpuAELdT4Gu6yUM5cywvl< z9=WdL?bbwvC*kUmsiOCbOQ+YqJxJ3TO zSkBzif{!ivbFf*3+gfwh1NV#ulX<~R6|}}4`PfpLsyL!mlzoksnc`0za5?&u9Js9d zi3{&5eo~AVrZ9DqWU-3MUw3VEChy@`_U}sT=JJYX^BUuMjdS)5bJ^8bT(YA9oW?SP zOceuc6_id_Ci3gwFR7Y-;(Oov6b}&>H&SJ5}ie$1HcEV+%xw)wgk$&2--cV<+vw+(4Wb|XVFTMU zWl%6veRMBlFb!mn?qhphhRcSTK6fu?a5l{Ld3wEu*M?nvzFxoK?=3UR?6tcu&>J*@ zz2!!E13F3#G&;vG&+7=jd3o(X0@nqak!EvlD8eg8;H>N@(HK=n8Nm}qf;WuR@{SUj zQGL|44hcTNFO(gv76M&Nal;574G(NxDXRz{<=T)?a}Wzbp}Y;-eN`7*+_Udf(O1)3 zYt(^8;eqF#!C&X-V^i$aEREN=$wrfG^vv|T!@vL6K(;(m`6sgU{ zL7rF4W}_)lNw;Ok_&?=;%oom>*9vFN_snVDyi)kSdChzWz`dew^bGX1njaPN=2i23 zK>yl&*Zk1DT9|C}9*AHoB^UeL+OJ5GD!(FX zGGG`&8=)+hoE}lqvcN2doYDSb2sf;A6R(M1<5_0O8EJ;dwBq1p&Ijt+1G_bR( zd_q=}*vv1qV^MmRyP zYdE8&L_;S+8?Bwu(M(E?reR3K^>kS~A*Q8VDw-M_&@#v6q@hQpOj2*vWl>9xMpqcr z@+T5vI-M~@Lrx^LRUmIQbUg$^9({%?uy>IR7v1*s7|_VR4gp9b9!G`svap+16l*`jr{s&chPH*f`RvE&?lg0 zzp{+8MHxIAs&@2tA*qR+m8DkPwZ_?9o&{!*`tse@z@4(x*Qx%W7H0)3s3^aaR&d01 zU_2ZX4e=O66CNg2)_DjTIZc+vxkK^(_?|(2>-gFw67cFYxG@A3Ue6_yvaS#3)baXf z?GKBJDoZ>0@us*Fl6X5#@g!u)QVYKnpmy?+pyq~aSnM$w!kfiuxpaaANsYLb#ie9J zlt4|(o77|yAw*TRTxwEVp!E~#d@<&$zUXUH1yP0JHMnUu3cVT+YDEu(23 zKx@P^BeVu-8(9qDw6Lrb*W1AJ8gUtmQRjq#?5+K8?w_mgDAae%20G`sPLu2Wj7H^aCnp{<-m@>Be$gzC z++{W|xLEeNd!d>)cK>MiT&M~DcSB8=jQNV{H(Gw!a?!Z-&Dnbl%;ue|E<~vva^Kf9WnGe8PAZ*TKmjFKIA2 zaem?-i++&x@v=-1dneTNIo!x!^#ozi3J&p~^PA!A(>>GM&8n_BuDif>f4)G&<`?=W zV1hG^dn@qIacnL33mTxyt+8hIdKHfG-1TaPz_mU=Utf=7jm-58H2~j0>*9^*#yT8F z+>J(>!0Xq>UF=O7$0_b6!w{JB0sdwP#~J2kWetH_w*l`h9)rwVjhMjAuGrT+w>INg zYv9%vFM)SZWbEc(cLrhLnxQaAn_+C&<%&0WX8brF49t{y3A~yjV=YH08-06&^i1pO zyrwNIIpS9BgcJq^{$+ejl7 zqgFu%+W{FZlQFbyupWf2lZ`xj;IEvLF7%lj_n5wz$;52>ad#B7SB=qU>o;L@{;y(y z=-*K)6S7d6A4a4Cl_1F0Lv8e|%sk+?GGqGuZP& z&!KqYTSEW18wwp>mkkSlLu&&z%U!&{n8lGWN22kge?g1TU_jk+L18DSPgty;RTaAm zl#V5Gnrd;nI4l!LCz?B<%T*Mn?FQTt==$?8j;N+3>N2Vq&jm?&A&F_7%Y0Kpk zzbt$0#N?@Sr!MT74f2;NejjYUx+C8^4`x;_)n5L#>Dys4J8UaQnDMGn!^rN8M%7GG zR7W$qv2%NSd;2rc-7C>sw{^9(!*A=Csh81&I>KA-L;{LKDE^4yEjNko@Gv0c7Yk)o zIi`_YTJB+}u*IyRkr!x%ORKijQn}og%xJns$~DWYe^(XNjTRRW>} zb@`G`7Do;cbH=C^A&U73m?(K#KuIUnoFqGI8sytA-~NP8I|lbWq|uxz@74xD5~;lP zFJOQOjqxHyVdk+5VecPM;6B>?7gTp2ZM%=^{)*QB5$%3}+;b>&2Zb(de5>WnmfxX` z^PciKPwgE~?S<}HPjlY25I|Jb6ACKr$233-1X}Ds?!dgaZh=Cp_E7Ww(0p0d6A$aA z^7KME!pt11D4>dYrs7kI4Y?NlC|oxms-9YJ4+~WYhn^rGrvBE5@RkRNE|O2}x{Dgf zY3$kX>F{fzsotrb`S2`WyGUUi$K>>e1sh!89HN>Fn=aH6qUJ@4!Ci!A{ig)7ALzdU D2Klo> literal 2199 zcmah~OKclO7@pau?QEPjPMd_J@usFq-L!QQf@mI16+3BTFjd&TBuKHg_B7en-Zi`H zDlwOoR%(wt4uuoFfLg%`A;C4tg`*)6fu)F6oO(-?iXO^=e|EFZLm<|kf4-gnJ@fyw z-QoHWg6AyTa_vbILVwa1|KT>l#$N#D(Yr{&X*7Xt9Hp2^GZU=Mx|mIK6THp4m`e+` zuuFD3pHGVu0Xr}ew1ZJ}8FwN@P{h};=gSUV4ox=u0c_1mpnp?B2{xntO@=LjblaAc zkCaeNx^A0v_&=njnsog(=>~<1p?LTyH5$i`cv`jW(aAJG&L*=q%mUG;HJzM|Gft>r zsQGNs$diK8JWcE@DPAG@yrxfQ^IDPUmS*Uc$e58U!;3ASBdVF3kv(&~t=R?Qh?cFI zcDa=Ev!HVT#w{2d4y1n&Md29Q&=lpk&m`lct+}(dJwrIlmYFh}W&900gB5g*MQ9e! zvl%{wO6Ve*^BJ_u1_m2JUw&(@`P?NRMN@UYl*0H{qaa0p{SX~} z&M-T@fL4RYLACsLSmLJk?&eFVZ-@OD5>e)T81KLshjCO!-JT~Xb#G5Z6hd8_hNfY6 z+mXgZ(E-?b`pcw-*aJwzGMbWnZvx7=Td%Sj zj`L?Lfpg=T@zi;_x6(1D+Ugb6B67~qb&|88cx0=b%Mr`ES}s)D|BZiD)e0m(C|6?R ze(A|>J<7B+N&cu@E5t!L9yIBxJL0ATM}s19wmV(zyR#_eb5I99xy5RVn&N+FNbxTajLQ9z;zq zgX*&|;u~&f9&y2{lvwFGv)q%eCSI?G-uRiztnUSxC;J9|m7>o@PaHIUio{+#3n6LW zGlU_IKz()3(FbomNL9lONKyjiDL>2~dvWr_7;wV0K5Ns@VV!(?r3^I#{T-xQX#Dx?S zTuk6(UwE-6MDY+q=}D@)gsJ}}hPqwi>r#i=r9I(Pj9u#B>3WQ(D~DOY4o>7C-r7Tt zo*v;w7|1;BMyxOjn_%D?G+pf)s@4rxxnc7qSgweP8KM^KnH_P_ za&zN4;c9wu{ofs^unO0|c{{op@aBF{cSO&@oFnBon=D&43yw54dVVxHdTu;>LCI7^ z-LPeg*l@8khUExwp%eXvo0l7>mzUs~G7o~?b{G~VUS`_;j&B>kYyGD6`;i|8mU}N$ z>n>Kgi>_{bk0V}%Pl;O6Oq!JwvfeAGh1?7&&UxJ@K|dV#qjUSzl&N5tuGhXR)>bb% zEO4BFp=Zer_`105lP3wV~l@CZSFt%8wz^h z4>a;bsINA~SA-+;>|@N`X}{TiS6jv{>r4p0i8olpw63{utpzdso#va(cO$pj*BOjY gQf7FKqBX(iwJzfbWnOYsn<)1ZUKhp~%(gxMF0Q* diff --git a/app/__pycache__/schemas.cpython-314.pyc b/app/__pycache__/schemas.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb9fa0a278e27616e0eb4701be7ae6fe8eb0200d GIT binary patch literal 6201 zcmb7I-EZ606~C0I?@!y3T|0@bI8Hh(i9IaYmUdaOEna6Qt&^~E+I4LRMxt#dvZ!8C zZQUn1Efy50SFb<-?`0eImN&0kuwdD;e?fdG;1K9TvAy)IiiLWQJLi&=Nr`bB0Xn=q z_ws(7-}#+$@9}ghPT=>?#S=G9K1W!ajZU3MZ@6!?esHsZJ@|>Bi;VjEc zBi1B+$qZ|{q}R2(oEg+~S`3)s1(hyKx|tHiQbp3VvQpATQNzB+8|F`@megf=stTf7 z!j;l@rE0mpJhi-XiK;i`lCDja)sl8nlO=y*xGXN z25W1OZ^C5aJ54^fgG>03P^@5W{o#26sy*<`S@2}gb?e|&x)I)7Gt!Vp0kg%8bW9mc^OHKR2E&U-2?(-c$`12h8H_rC{H*z~`S<4MMx(iDeB9h@-ZO3)tH!5==D$I8ztDUD^?z;NZ~m)!uYm6l zn}31w!{+Bua}P@QL229q#Xs2V=A;?E4t^+8e{;VCV?L|7ZE6bIq{CaLPfDedtZAZ- zYcf+*UZAqJWWUClR@7H&@;>k|;@KAu63~>|cR?^DeAS;f_`E{}xgmoebZDS2XYiwp z2VBr7#{8gRMnn+^3w$0u>uErF9^0T`Q$eILAAZEY1#Ubv);Pa| z;>I_~@PL_+IUraYYi|+rN|uw%DZ7o&crqmwv=r?@umG(}N> zzlJ&)Zjlemjd z5<*B%LJE;$XO6I$br$YeWX>}U&YA`3CW7f6A{grK$V&zk6LH21yIxKiY-gZi8Y^E! zkwfuh-~JY=_Y-3v#poad%TGT2^whAk5PTB=J_ut>v&#{?+}l^wt_k+^6?@`0al3|x z-pL6>mHq4KBY`ae-AOBEo`p9vq$_#_&SGH<%PKV!b>>wv#Y3C@B9`HVWG3MJrHgV| z(dDukli%0ns-~z_Gp;Eg$l~=CIDD9ivaFS;Qq!>_qiCY^uB23?>u}1^)naIu^~@;t zsVk742bHRB@)bA_?Cn^GdSKx?5s7DDG1?zMST4orka-jk;CY7_Q`z;2wetpl$YIC+ z!43Y-0fRr%!c3_ZC;f1s8*U}=C27APk+sq!JFtFb?E`}^v@)p3lD^z}`A*#654ZYI z(NFsG>-rt|e!Mk+iX0gn`TfM5H~%#8MgEwXdOb683*9@oy0CC?R=mMlL&hkCKJAx^jX-` zQiL~f!6?{#3)XxaWb3(ujbCgH9|9ci!p?n2!B0G3>qIZ?$OE>EUasQB=a%Z#o6i=iUhfH2IZrS9jZ@cPDX+tSUvcWU&~Pu%Lu~Q8rR(INcFb{L z9qfU1sLM4vkxJ;UVQ~xMM|3xS&~q?7Gf~3L6qRaOe&6JEhz)aQ^rpOmmoAzatw`0y zy0j=4c`E>1!L@h%VsDoi@$Rsj@{u65H$ik$zLigPJiy>{4&P%o>`;AX;CH9il#Pim z(xV1{)ZzS0|N8XW4TB$hjJ)r1jkvFCT*sv#VE0AMZ(&0eZ0h4su&6ms#*Z~-whoLn zW?UqUVZ|X~{AjM2g1fAhvQ&jDKN{k1`WM;+9_@nxF1zt7Y@cn%Q=ngI6L_=>JU6+)`kZX1W7yBP1RiYD F{|}%#7Ki`< literal 0 HcmV?d00001 diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 0000000..b580143 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,283 @@ +""" +app/auth.py — Аутентификация: JWT (access + refresh), хэширование паролей, +зависимости FastAPI и роутер /auth. +""" +import os +import logging +from datetime import datetime, timedelta, timezone + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm +from jose import JWTError, jwt +import bcrypt +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select + +from app.database import get_db +from app.models import User +from app.schemas import ( + UserRegisterRequest, + TokenResponse, + RefreshTokenRequest, + UserResponse, +) + +logger = logging.getLogger(__name__) + +# ────────────────────────────────────────── +# Конфигурация из переменных окружения +# ────────────────────────────────────────── + +SECRET_KEY = os.getenv("JWT_SECRET_KEY") +REFRESH_SECRET_KEY = os.getenv("JWT_REFRESH_SECRET_KEY") +ALGORITHM = os.getenv("JWT_ALGORITHM", "HS256") +ACCESS_EXPIRE_MINUTES = int(os.getenv("JWT_ACCESS_EXPIRE_MINUTES", "60")) +REFRESH_EXPIRE_DAYS = int(os.getenv("JWT_REFRESH_EXPIRE_DAYS", "7")) + +ADMIN_USERNAME = os.getenv("ADMIN_USERNAME", "") +ADMIN_PASSWORD = os.getenv("ADMIN_PASSWORD", "") + +if not SECRET_KEY: + logger.warning( + "JWT_SECRET_KEY not set in environment! Using insecure dev fallback." + ) + SECRET_KEY = "dev-secret-key-please-set-in-env" + +if not REFRESH_SECRET_KEY: + logger.warning( + "JWT_REFRESH_SECRET_KEY not set in environment! Using insecure dev fallback." + ) + REFRESH_SECRET_KEY = "dev-refresh-secret-please-set-in-env" + +# ────────────────────────────────────────── +# Хэширование паролей (bcrypt напрямую, без passlib) +# ────────────────────────────────────────── + +def get_password_hash(password: str) -> str: + """Возвращает bcrypt-хэш пароля.""" + password_bytes = password.encode("utf-8") + salt = bcrypt.gensalt() + return bcrypt.hashpw(password_bytes, salt).decode("utf-8") + + +def verify_password(plain_password: str, hashed_password: str) -> bool: + """Сравнивает открытый пароль с bcrypt-хэшем.""" + return bcrypt.checkpw( + plain_password.encode("utf-8"), + hashed_password.encode("utf-8"), + ) + + +# ────────────────────────────────────────── +# JWT — создание и декодирование +# ────────────────────────────────────────── + +def create_access_token(data: dict) -> str: + """ + Создаёт короткоживущий access-токен. + TTL = JWT_ACCESS_EXPIRE_MINUTES (по умолчанию 60 мин). + """ + payload = data.copy() + expire = datetime.now(timezone.utc) + timedelta(minutes=ACCESS_EXPIRE_MINUTES) + payload.update({"exp": expire, "type": "access"}) + return jwt.encode(payload, SECRET_KEY, algorithm=ALGORITHM) + + +def create_refresh_token(data: dict) -> str: + """ + Создаёт долгоживущий refresh-токен. + TTL = JWT_REFRESH_EXPIRE_DAYS (по умолчанию 7 дней). + """ + payload = data.copy() + expire = datetime.now(timezone.utc) + timedelta(days=REFRESH_EXPIRE_DAYS) + payload.update({"exp": expire, "type": "refresh"}) + return jwt.encode(payload, REFRESH_SECRET_KEY, algorithm=ALGORITHM) + + +def decode_access_token(token: str) -> dict: + """ + Декодирует и валидирует access-токен. + Выбрасывает JWTError при невалидном или истёкшем токене. + """ + payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + if payload.get("type") != "access": + raise JWTError("Wrong token type") + return payload + + +def decode_refresh_token(token: str) -> dict: + """ + Декодирует и валидирует refresh-токен. + Выбрасывает JWTError при невалидном или истёкшем токене. + """ + payload = jwt.decode(token, REFRESH_SECRET_KEY, algorithms=[ALGORITHM]) + if payload.get("type") != "refresh": + raise JWTError("Wrong token type") + return payload + + +# ────────────────────────────────────────── +# FastAPI Dependencies +# ────────────────────────────────────────── + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/login") + +_CREDENTIALS_EXCEPTION = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, +) + + +async def get_current_user( + token: str = Depends(oauth2_scheme), + db: AsyncSession = Depends(get_db), +) -> User: + """ + Dependency: извлекает текущего пользователя из access-токена. + Выбрасывает 401, если токен невалиден или пользователь не найден. + """ + try: + payload = decode_access_token(token) + username: str = payload.get("sub") + if username is None: + raise _CREDENTIALS_EXCEPTION + except JWTError: + raise _CREDENTIALS_EXCEPTION + + result = await db.execute(select(User).where(User.username == username)) + user = result.scalar_one_or_none() + if user is None: + raise _CREDENTIALS_EXCEPTION + return user + + +async def get_current_admin( + current_user: User = Depends(get_current_user), +) -> User: + """ + Dependency: проверяет, что текущий пользователь — admin. + Выбрасывает 403, если нет. + """ + if current_user.role != "admin": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin access required", + ) + return current_user + + +# ────────────────────────────────────────── +# Роутер /auth +# ────────────────────────────────────────── + +router = APIRouter(prefix="/auth", tags=["auth"]) + + +@router.post("/register", response_model=UserResponse, status_code=201) +async def register( + body: UserRegisterRequest, + db: AsyncSession = Depends(get_db), +): + """ + Регистрация нового пользователя. + - Если username совпадает с ADMIN_USERNAME и пароль совпадает с ADMIN_PASSWORD — + роль будет 'admin'. + - Иначе — роль 'user'. + """ + # Проверяем уникальность имени + result = await db.execute(select(User).where(User.username == body.username)) + if result.scalar_one_or_none() is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Username already taken", + ) + + # Определяем роль + role = "user" + if ( + ADMIN_USERNAME + and body.username == ADMIN_USERNAME + and body.password == ADMIN_PASSWORD + ): + role = "admin" + + user = User( + username=body.username, + hashed_password=get_password_hash(body.password), + role=role, + ) + db.add(user) + await db.commit() + await db.refresh(user) + + logger.info(f"Registered user '{user.username}' with role '{user.role}'") + return user + + +@router.post("/login", response_model=TokenResponse) +async def login( + form: OAuth2PasswordRequestForm = Depends(), + db: AsyncSession = Depends(get_db), +): + """ + Логин по username + password (OAuth2 form). + Возвращает пару access_token + refresh_token. + """ + result = await db.execute(select(User).where(User.username == form.username)) + user = result.scalar_one_or_none() + + if user is None or not verify_password(form.password, user.hashed_password): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Bearer"}, + ) + + token_data = {"sub": user.username} + return TokenResponse( + access_token=create_access_token(token_data), + refresh_token=create_refresh_token(token_data), + ) + + +@router.post("/refresh", response_model=TokenResponse) +async def refresh_tokens( + body: RefreshTokenRequest, + db: AsyncSession = Depends(get_db), +): + """ + Обновление токенов по refresh_token. + Возвращает новую пару access_token + refresh_token. + """ + try: + payload = decode_refresh_token(body.refresh_token) + username: str = payload.get("sub") + if not username: + raise JWTError("No subject") + except JWTError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid refresh token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Убедимся, что пользователь ещё существует в БД + result = await db.execute(select(User).where(User.username == username)) + if result.scalar_one_or_none() is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found", + ) + + token_data = {"sub": username} + return TokenResponse( + access_token=create_access_token(token_data), + refresh_token=create_refresh_token(token_data), + ) + + +@router.get("/me", response_model=UserResponse) +async def get_me(current_user: User = Depends(get_current_user)): + """Возвращает данные текущего авторизованного пользователя.""" + return current_user diff --git a/app/main.py b/app/main.py index 042a32c..43280e9 100644 --- a/app/main.py +++ b/app/main.py @@ -1,169 +1,94 @@ -from fastapi import FastAPI -from sqlalchemy import text -from app.database import engine +""" +app/main.py — точка входа FastAPI-приложения. +""" import os +import logging +from contextlib import asynccontextmanager + from fastapi import FastAPI -from app.database import engine +from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy import text + +from app.database import engine, Base from app.embeddings import load_model, get_embedding_dimension -app = FastAPI(title="Document Search API") +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) -@app.on_event("startup") -async def startup(): + +# ────────────────────────────────────────── +# Lifespan: инициализация при старте +# ────────────────────────────────────────── + +@asynccontextmanager +async def lifespan(app: FastAPI): + # 1. Создаём таблицы, если их нет (без DROP — данные сохраняются) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + logger.info("Database tables ensured") + + # 2. Проверяем подключение к БД try: async with engine.connect() as conn: - # Используем text() для явного указания SQL запроса - result = await conn.execute(text("SELECT 1")) - await conn.commit() - print("Database connection successful") + await conn.execute(text("SELECT 1")) + logger.info("Database connection successful") except Exception as e: - print(f"Database connection failed: {e}") + logger.error(f"Database connection failed: {e}") + + # 3. Загружаем модель эмбеддингов load_model() - print(f"Embedding model loaded, dimension: {get_embedding_dimension()}") + logger.info(f"Embedding model loaded, dimension: {get_embedding_dimension()}") -@app.get("/") -def root(): - return {"status": "ok", "message": "Backend is running"} + yield + # (teardown при завершении — при необходимости добавить сюда) -@app.get("/health") -def health(): - return {"status": "healthy", "database_url": os.getenv("DATABASE_URL", "not set")} -@app.post("/search") -async def search(query: str, top_k: int = 10): - from app.search import search_by_query - return await search_by_query(query, top_k) +# ────────────────────────────────────────── +# Приложение +# ────────────────────────────────────────── -""" -import docs_parser -from fastapi import FastAPI, File, UploadFile -from fastapi.middleware.cors import CORSMiddleware -from schemas import ParseResponse -import os -import uuid - -app = FastAPI() +app = FastAPI( + title="Docs Search API", + description="Корпоративная RAG-система для работы с приватными документами", + version="0.1.0", + lifespan=lifespan, +) -# Configure CORS +# CORS — разрешаем фронтенд на localhost app.add_middleware( CORSMiddleware, - allow_origins=["http://localhost:3000", "http://0.0.0.0:3000"], + allow_origins=[ + "http://localhost:3000", + "http://localhost:8000", + "http://127.0.0.1:3000", + "http://127.0.0.1:8000", + ], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) -@app.get("/") -def read_root(): - return {"Hello": "World"} +# ────────────────────────────────────────── +# Роутеры +# ────────────────────────────────────────── -@app.get("/items/") -def read_item(source: str): - try: - # Check if file exists - if not os.path.exists(source): - return {"error": f"File not found: {source}"} - - # Use extract_text instead of get_text - # extract_text returns (text_content, images_dict) - text_content, images = docs_parser.extract_text(source) - - return { - "source": source, - "text": text_content, - "has_images": len(images) > 0, - "image_count": len(images) - } - except Exception as e: - print(f"Error parsing file: {e}") - return {"error": f"Failed to parse file: {str(e)}"} - -@app.post("/upload", response_model=ParseResponse) -async def upload_file(file: UploadFile, - parse_immediatly: bool = False): - file_id = str(uuid.uuid4()) - file_path = f"uploads/raw/{file_id}_{file.filename}" - os.makedirs(os.path.dirname(file_path), exist_ok=True) - - content = await file.read() - with open(file_path, "wb") as f: - f.write(content) - - parsed_path = None - parsed_text = None - - if parse_immediatly: - try: - # Use extract_text to get the content - text_content, images = docs_parser.extract_text(file_path) - - # Save parsed text to a file - parsed_dir = f"uploads/parsed/{file_id}" - os.makedirs(parsed_dir, exist_ok=True) - parsed_path = f"{parsed_dir}/{file.filename}.txt" - - with open(parsed_path, "w", encoding="utf-8") as f: - f.write(text_content) - - # Also save images if any - if images: - images_dir = f"{parsed_dir}/images" - os.makedirs(images_dir, exist_ok=True) - for (page_num, img_num), img_data in images.items(): - img_path = f"{images_dir}/page_{page_num}_img_{img_num}.png" - with open(img_path, "wb") as f: - f.write(img_data) - - except Exception as e: - print(f"Parse error: {e}") - parsed_path = None - - return ParseResponse( - file_id=file_id, - original_filename=file.filename, - parsed_file_path=parsed_path, - status="parsed" if parse_immediatly and parsed_path else "uploaded" - ) - -# Optional: Add an endpoint to convert files to new format -@app.post("/convert/{file_id}") -async def convert_file(file_id: str, new_format: str = "txt"): - # Convert an uploaded file to new format - # Find the original file - uploads_dir = "uploads/raw" - original_file = None - - for file in os.listdir(uploads_dir): - if file.startswith(file_id): - original_file = os.path.join(uploads_dir, file) - break - - if not original_file: - return {"error": "File not found"} - - try: - new_path = f"uploads/converted/{file_id}.{new_format}" - os.makedirs(os.path.dirname(new_path), exist_ok=True) - - docs_parser.convert_to_new_format(original_file, new_path) - - return { - "file_id": file_id, - "converted_to": new_format, - "converted_path": new_path - } - except Exception as e: - return {"error": f"Conversion failed: {str(e)}"} - -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8000) - -# NOTE: все эти точно работают и работают хорошо -# print(docs_parser.get_text("parser/assets/text_and_tables.docx")) -# print(docs_parser.get_text("parser/assets/some_text.docx")) -# print(docs_parser.get_text("parser/assets/text_tables_png.docx")) -# print(docs_parser.get_text("parser/assets/text_from_img.png")) -# print(docs_parser.get_text("parser/assets/main.typ")) -# print(docs_parser.get_text("parser/assets/main.pdf")) -""" \ No newline at end of file +from app.auth import router as auth_router # noqa: E402 + +app.include_router(auth_router) + + +# ────────────────────────────────────────── +# Системные эндпоинты +# ────────────────────────────────────────── + +@app.get("/", tags=["system"]) +def root(): + return {"status": "ok", "message": "Docs Search API is running"} + + +@app.get("/health", tags=["system"]) +def health(): + db_url = os.getenv("DATABASE_URL", "not set") + # Скрываем пароль из URL для безопасного логирования + safe_url = db_url.split("@")[-1] if "@" in db_url else db_url + return {"status": "healthy", "database": safe_url} \ No newline at end of file diff --git a/app/schemas.py b/app/schemas.py index 4504b4f..4000aa9 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -1,8 +1,88 @@ from pydantic import BaseModel +from uuid import UUID + + +# ────────────────────────────────────────── +# Устаревшие схемы (оставляем для совместимости) +# ────────────────────────────────────────── class ParseResponse(BaseModel): - file_id: str - original_filename: str - parsed_file_path: str | None = None - status: str + file_id: str + original_filename: str + parsed_file_path: str | None = None + status: str + + +# ────────────────────────────────────────── +# Аутентификация +# ────────────────────────────────────────── + +class UserRegisterRequest(BaseModel): + username: str + password: str + + +class TokenResponse(BaseModel): + """Возвращается при логине и обновлении токена.""" + access_token: str + refresh_token: str + token_type: str = "bearer" + + +class RefreshTokenRequest(BaseModel): + """Тело запроса POST /auth/refresh.""" + refresh_token: str + + +class UserResponse(BaseModel): + """Публичное представление пользователя.""" + id: UUID + username: str + role: str + + class Config: + from_attributes = True + + +# ────────────────────────────────────────── +# Управление документами +# ────────────────────────────────────────── + +class DocumentUploadResponse(BaseModel): + document_id: UUID + status: str + + +class DocumentResponse(BaseModel): + id: UUID + title: str + author: str | None = None + uploader_id: UUID | None = None + upload_date: datetime + last_edited: datetime + extension: str | None = None + size_bytes: int | None = None + description: str | None = None + is_available_to: list[UUID] | None = None + + class Config: + from_attributes = True + + +class DocumentUpdateRequest(BaseModel): + title: str | None = None + author: str | None = None + description: str | None = None + is_available_to: list[UUID] | None = None + + +class ChunkResponse(BaseModel): + id: UUID + document_id: UUID + chunk_index: int + text: str + keywords: list[str] | None = None + language: str | None = None + class Config: + from_attributes = True diff --git a/app/search.py b/app/search.py index e0eabf9..b2f7039 100644 --- a/app/search.py +++ b/app/search.py @@ -1,4 +1,4 @@ -from app.qdrant_client import semantic_search, hybrid_search, delete_chunks_by_document +from app.qdrant_client import semantic_search, delete_chunks_by_document from app.embeddings import get_embedding async def search_by_query(query: str, top_k: int = 10): From 4c818076079d953f01102c278526e7d7e0b963be Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sat, 13 Jun 2026 00:31:12 +0400 Subject: [PATCH 17/34] Update front-end; Add: Docs CRUD --- .gitignore | 2 +- .../document_processor.cpython-314.pyc | Bin 0 -> 3702 bytes app/__pycache__/documents.cpython-314.pyc | Bin 0 -> 12282 bytes app/__pycache__/main.cpython-314.pyc | Bin 3985 -> 4276 bytes app/__pycache__/schemas.cpython-314.pyc | Bin 6201 -> 6228 bytes app/document_processor.py | 5 +- app/documents.py | 209 ++++ .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 821 bytes .../collocation_deduplicator.cpython-314.pyc | Bin 0 -> 5623 bytes .../collocation_extractor.cpython-314.pyc | Bin 0 -> 10286 bytes .../keyword_processor.cpython-314.pyc | Bin 0 -> 8792 bytes .../__pycache__/keyword_set.cpython-314.pyc | Bin 0 -> 23278 bytes .../language_detection.cpython-314.pyc | Bin 0 -> 2932 bytes .../__pycache__/lemmatization.cpython-314.pyc | Bin 0 -> 3455 bytes .../paragraph_processor.cpython-314.pyc | Bin 0 -> 4646 bytes .../__pycache__/stopwords.cpython-314.pyc | Bin 0 -> 14746 bytes app/main.py | 7 +- app/schemas.py | 1 + front-end/index.html | 1025 ++++++++--------- 19 files changed, 705 insertions(+), 544 deletions(-) create mode 100644 app/__pycache__/document_processor.cpython-314.pyc create mode 100644 app/__pycache__/documents.cpython-314.pyc create mode 100644 app/documents.py create mode 100644 app/keyword_extraction/__pycache__/__init__.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/collocation_deduplicator.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/collocation_extractor.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/keyword_processor.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/keyword_set.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/language_detection.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/lemmatization.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/paragraph_processor.cpython-314.pyc create mode 100644 app/keyword_extraction/__pycache__/stopwords.cpython-314.pyc diff --git a/.gitignore b/.gitignore index b3b3ee8..c14bf69 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,6 @@ lib/ *lib64* target/ share/ -.env +.env* pyvenv.cfg tree.txt \ No newline at end of file diff --git a/app/__pycache__/document_processor.cpython-314.pyc b/app/__pycache__/document_processor.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40dce9d17a3892d7908ace7bff0ecde7e69cf48e GIT binary patch literal 3702 zcmahMTTC3+_0G;dW?!?*a}6xsSr*$Y*bqBV{H~lBylXHa>v7boYmdge1B}_nD%qb~r5{nhsFX@c)Wm#ZLj{?F+RFZ@|8TM7w&_RDod=5}xxLHW zd+xdCanHHuo^$D8gFXa}o;rB#4_<_xkcr(~7O=epz#O`QGE^FAWOSsdltXjG2u!Ex zlv8sWuruvSxiz-|yV9N%qcH~TPJ2^6&1b-#v_BQl0tU>agDF;H4VX%Yw9w_yIA^P` zjd66juAegJTK#2T9A&&$mNLFUhlOjn9EhWSi$QBNcG{#hWr7)h1Z4sdR0tO+tvN#_ zP?CK{=*E(iUUx>lQOaGER8=mQM$0*|07%!{QBxOQYHL>Sjtz5eZ?DK`ZX|D?E?+!AypD_ZmLgI z6f!q8Dz?LVl^c3M5X4fctcjW=2)GHPH^8Uvhi(qNL~l|&_1x%xOK%zDXV&;f&BSj5 zCCU-!*ewje&*0m>>4d$a7zk=ZHJ6O=2uQb=;pdE(W0(57y{xh^K98 z#m&19Sm?<%avWK5SVkKU*-&lF&^!vDBMzHC2G=QpvUN6v9mXdpfvz|zwBWEY=ctV1 z8rXTAnxjWuKBRcE#8pP)-L||i8lr{)Z&{FH%?~s5rCW5wq>m z7Rp9#?D$?As*Sa)tc0_18_$P#adp_(+89SXvhCp+-p8}NpZAXMt5Mic#(f3#TQTq& z3Qvt9!wfPer&7mz#_glFg~7=0B&QexisY^(3*0$DIP2Xf7(>|}n>u^QhWPM~9cagr zP{un5ekM(+=j-ilmaCLx-lNCw_^$Ts9hG?;P+Xqif~vyH2>#5w~K3@GW*8H?K#hTDPP5Q51S#W5nzyHfyX z&9^5|kX` zQB1-_4@}PpJJm;Twe{Iv5+-oDXlTbI#&u_<0@cw)`hHBJPD|@EFi(F%N4K zg^D_b$x{iF6t25-l%VFySWF1hn6F$RVMun!#)jhYoQMKTMczQ zxVjPRs!dbTQuq% zp|+8ulU)vQ&)q0_YBh4|QP;_PmAm+32LSV_N0C!c_w0LcZDr5l`SjCB+nw28&VKac z`P4>Z%X&lGMwGjs{#E+H^h)%|dNj7)6kBhJlfSlo_fOwDT|MyW;gz-l*vb99_x4s@ z)%RCg4!@}LHnH&bJFE^cBWln7}?6+GkAo=${>VflL+hMfl z^m7gc+n+~~xAo-(OCE?^33dEO{T&?sPvPB9-P{97@Yhj*e*Xq^UnHqiBlAT!P2fXk z9I1rs%f4VL68Q4uTP#ez@=>XF-&g)l0{6cG{nwpTD#m=hFA4BN)(?CSTc}jr`LNYZ zV9uYaqaXIPr2_Pln_!j#F2F3+ISCx`0M}BCB7K}}XDQ)Mo$@ZhE)KFw-97>jI7mO} zh1DcAmW*Ih)${;7J$uQ`x++4BF|vY>po-CP?f0304MyB-OisO9`)|Hz_3UC| a)pvT%^GztW5{l0So}l`d&e|j1Q2JkLzF)Wi literal 0 HcmV?d00001 diff --git a/app/__pycache__/documents.cpython-314.pyc b/app/__pycache__/documents.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6976aff074a662dd38bb678ac5900bf09f4692a GIT binary patch literal 12282 zcmeG?Yit|GnX}8~`zeW{-V*h&EK0JJ*h;L}P90g6Oh%5)ur%W~lzFtcmS{^Pv%8FK zHEHDJfXLSrv6~c@&jAwPih~RksC!qSIb4Bra#rHr14o)_DQ^|n#pQ6wKPomZP4efy zZ^>OsG@VDUE&Ag|;>^)x5Ldbt&Mh;R9WbSY(LgvU3 z;-fsoO$n5n7HGFhP`Md_ajONjTO(-PT0z@_IJ8IS)(d(GR(T9=qhOR^#$$4u1+xUJ zJr=-utBQvRJbdJN(o2nsdC!|ySrMbb~^-zRHpOPxNC)4cb!n@t{3Xv z4MKxduJ<&$n}j9_Hh7xdEkcU~8$C{UtI*m)#;I1~Gx^MYl+UuA&JR#%^I3)VajVxY z54f1NlJRz*%~v7g0);m3Sg~G)e3j#MzA71OTn)Y02dI4QxCV0S-y=su?@dZO%vH*{ zZCYP-4e>c@$h2*m8Yr+KbOLt=WuA(uuV%b!ytcP~T}hGGFY1N%)j}_ICB3-r(@Xt$ z?YcSmV!91Lx3Pq7_kHL#0rw5R7x&H3OG`;FJ><(A@%b$7O28 z)K@Qmv+*yUkv5>&UP80qr*0>%j*pO>ODpP!ydGaXDX^TV-ODD}ScDhNZol6%{&Z~On=H>mL&$4* zcAA|K08+)d&;cx&WQ9=Vs7oa>hj{2zw>LhKoMB^vs2-k5#!k2x(Iyx9SUwSt@vP{O zOY)p3<<-UVNLXNf>@!K07euo}NydUsZP@8VlH;JCP!hVuvIG~O07gTR0s~N17Xfw! zHZ*~p@S&rpmC9K|zl_I=dIfaRq9(zy$D*f1MhH*xv&N(}SA0JgKIsRFI`&kQ7eet9 zv)cZA6%<;y!oWf^apL1AQQ=*D-g|*069Vr{v{z1C~G87Qd5+V3PMdj z_~>9k;t=9f1sSEB>D9_G)Tr-Uqq%R5_P#Z`phl+c(+71j+^>UmW%vmB5p746r4bTv zZHUYUDQh8BH94yU+K9e-tpdS2&?}ix<{C04_`L+bJAVgXy^XX0Q^j@RRaOOH6hbSJ zWqLx_M-uvgdUK&B@-(ikfZD5;af_*s$g;*pMn?!q7y<_Gy0s-^Dzu?qraLLE*|8=< z8rJi+eiVh~tt4QO`)MHoz1JwyEvD-+#`Y1SC(R^4d2zp#gq_qz4qAwhNf?8bGMd*e zL&emKAGag~j51FrVLrKk513`T5vMd`W^Y|d6UY|zvbW*hT6vzHpnRHy ziVfy?lZ;zT<#Fr-%j!6M=Tp?2YQNe*5)}cPS7xk;P68D&7cC^{k}Ha(iAtXa{|0~- zh8E(d@BNN`W5OQXlE)y#uk+jeD!<0tCzljczvXbHpYdD$dYPHL1oi82=6#0kC`ceA zs{LB;gL!y;?5{5J3mDth*X7~+%>Zytybs-rAADFYD}Ecl@@@Q9pK-e~_TVEju3v|? zrECSZ6DJAc&XvQyPWqLh~Gzu_c3M4ppL zSP_jRO1F?Epn`wN_CPCyz$&N{vUMkc~KnxRVb*%Q{ib>~OmDA{~uz$RF1Rkyy~nEEkGKL_K>-U}I>a zhz347%Yvd8Sf0ZjA?lGpC=nK>xDLpJf;YS_|1 zWD>eiV>}?L`NVV-OdC*D(F9mLFhJ0K2{f}rLm)hzWJfqI&T%MsMICk>3(v439S4Pp zJcv4p0Prhnrt?J9)A5sFH!;WKQ8dlG`wtH74ej;#IQ*s}69?KN!?9uD;UpIoSWyK; zMZIFqakx}CT)d(x9EpgUiTKP6@Th}~q^mgxBA~EzIXL+4gmYq~%%wC>(&TU`7r0K0 zq)D(AwKp1Byr_an;D(`Iq(Y(!M#148$WJ15Ow@v2h7Tcd+a(lMGOR+9i6H9aS?7_R z&O&f*2pq~0(VDlb6jO~uHHeM)&tZXl1bXqV161mT?jLl&);l+r(~z3Fxr4W~j+Lsm zY*kmfs%x?8*Hv5QhgT}AFSyRTUhA13zGbaE%jamOB1Em)YcF_y;8~b>{lJR7J!|hy z+q)OH{o3BYW+e8$oSoS0uReeF`K+}qWo>(lo^!97t*<_H_NlD7DQ#|An97(ppBbGS zy4BNnPIbX>-mtLkHEY(kDQ(-dY}R+Vk2Art_vZzWpC{i(`wAzD>RD zdb=U*8d*O4bSgNRIXsoIMdv&@TB8n7tLBPxt_6SA+_r3Pdn>%s=zP<2(Q|1m>)1TM zZ^dMOb>G>2=k_g|8dvKY7oNJ*IRE%3T4HI>*+_*WYimu}S{G{PJge546{BOtQnO;M zUa>nb96Wz;rLtkAs%gdE0>G`F9T%-vnY&d6n`X|Pt0ijf%cEyT=LeS=`-;Kzs_U%l zdp%jEI>l7Ol&Y(QX4n4I;cyD_%+U{xt&1nF_AVO_rkI1jyZZ!1 zYTND-3a7Sy^OfP$o~KeU#;1a*$HtfIkEHBJaw@F+Y^|2mbbX=%#$lkJ-_;RQH8jKF zTN)NssfWk@Z8+t9@?S@<52gnE%a%ha-J#E+DIk2d)%Zo6)Yb|T!SgYu9W4F7%_3~v>OhCy*hf?!%_8*^PynKz83O(U7JQWU_7 zMybb7h>faV!)UeAkyAnXq0YXhhLr08go%Rn03ab~a=!$>JFmbVD_EZZUIXhFXOE3YZoq%TVkn}1o zs>6|)XzYN?%%K%6$$QWUadf{KdZVD{CVi7q0ECL8QbU`waRR{qf%EG=G8e1u{$p8IKN|o z%~W+{%^h>2D>lak%Xv%2=FAh#*gEGtD@J>Yft5}yn?B`H@x8dGVVI%bZK8&0=H2E^ z0MF|CzXJmS6X5v`n-)D$wA$`?A^NgSTh1#34_?_9;K3`Nb-${eTxO(A4W^IOb8+&a zVi)wsJ%-U&+UCarlShLNT~{k*U&CnCWP86hK)$`p9Pl_?DYg44D$UR%AjN7<=n4 z%vmD|n&qC*XkKqF!}cvBDtwG2tU{8oV#+z5(5?*sQmeg%R=B)Mg3MI}HWzTE2#)g9 zZy8#Tz}JYBj!q2@&pKzIATItI#l>aqfsbB?HlJ1QEgXY6;m8Awa5&WC`hZam zhD$zr4NzvyLlbyl=8UkhC>wDZxe2USWDFO^tYlbL=A}j`9%Dmr?iYhpGI6=TKom*x zbRdKy2cdETVMVfVpbZEIM-s9;0fCa2Cno9au)rtB14b0FE<=RUBxfpY_cxCH=y0aF zFWWqjZXWpA;h#VK)2FYFrXPAd(>#``+MhM=2RUG^&RVKSJ7sd;Hdf6YT&pIQC#X+(w5VSkqK8|lcNl88g?UHafN<-O z0g`un2OyrcDZ-{de-!zrlQ|qp3T_%A7j=KD;d!W6#EG5iS|boC;sjtxoIqL-Cjfnc zIQbXQpLrYN|HTPRtP;u#fImBN@?|OjeK9G>qN}V5P_VpcN|c0+DxloCTCmt2t=WtR<1+aYlPZg4;I)a6scScDqR)#m5v+-_4gHr9G++9 z-DzE%i@<@!(bJ+)k^-SUfYK;Z-(Z!!u+JnC<7q!s(D zYP~!yTzYV`fb<;vcn!#WNvv8(ZT*GW^Rqwd{BdojZELn|Fx@uzi`su`_=kq;n)D+F zGHnMlHQsE6ckb~OYhBjbo3{35tbKoUu?lWyedC+;7wcDbmRFMBGiP-TXBcQr9O=gG8Qox3H5UE7r1^Ud_0Z)SAtKzh!lYj@Q3EMoElOP_;7upMM#I1q#mHW=x3m{9285k4d$dE`c;26 zmLxb-kk2dx#G9a@(W{p{`n=Vj{TuMp|H zKaPzY(j@6VlB7pr<`SBqaw!wg$m-w#JqF(ZFO-sWLIW2Ba2Xa;juXH&FLMziuZB`p z0@qina;hYtJl81GLS0ZIyOVGfk32-tq2CNYo8;7M4{&jqP%eQ{A4X`FbI(9DyX|0X z`m|G+Vx94mU}rnyoHHDAN@oDh34oFbyc-9ED46c@UAOJ)sV7u_(n0#V!(W_y?n#JH z${Yi&Gv(6e6%wkCqQoiM^CFexwNQtLxr!2k<6GoUIVRxd0ONJ|@edajP~MIIquccb zCm_0PIiAuTzpb^r{QQ~cZ)lw>bbrIGVS}* z17nNN{AhTQ`rAs^l|7%3`Hg_!fU^#}Jf?Dbquilq`BM*h1{LaX=hujj z4vO^GJyetXghvNud6cQawW?zFue&b4=OhXwK3(pLJiy45O24esLS|NG2U7XV-izrdw_y~D}vgu#j0d~w8O|9ZM=5E^rPF&aTK z8tDZVj|bH=@n=~MMWd)ra8bB-#5W)L18=xp#+P4`KD)=SDrzw9ThQVjKgybKTvTE6 zfc5u?iN|q&+M}tBDh&J^oh_@aga4Ga?v0~wPF|dRGj=hycsRRdcY4e2Z1ZC`njeFU zPW>pgYO=iQ`76)4sjQmGKSZ2C`meGvYe&^g7Lla#0!S!Cr zMc{3d{aiFLX zy;;*p+BEWcPK{`vt=XX0n@shm{3J{vYB7`Kc2!DR4B*zG8Yp`Akv@q3+2tImCfD>` z0AG8M8r3n^wqos)o*FeUO9nl{)y7e^YRTC!N>ocb)ll*tQA5dlY6f8g1+nV^QF|;5 zm-vY&zDOn<%J00Bj=GN1VuTjC^Z-qC@Xt(#rzfV^nbUo6t=$*KXOGdi^b$?1SXX&0 zDJ0>>1*JH=HBcJmuV5rc3|A^VADBuC@VLPQZv*AyN>KHIDEGKTa3*|$jYQ!-h)KlZJwgx*Rd9?TGI-Y`-S9{Uv(hnw zs7fRS$)4k021E|G#bqLC;rxNFZWjzJIn929y98CRR|x-)5W_7lMcpDtvgAmb9QlCs ze?U4uAoaf?#@`VAZ;1K7NXsqKnwHW2o9>%+&Q$AznYyhvYnm6HeeH$+$-l`{5qftHUMYF0 z+;bFCu0K;hzh^;6bqxHf<)OLy4E6ADOIzY~>K$mfJx2qS>y(>~rMd>O#UsE_ee1^F z@HPh9?8wmofeJ6hs!mDb60G*m&(xq=KAjKO?OXk=&=`i11Pr( a+cceNn*UCQYADwS7*>eh_%SV;SpN&-5!XWi literal 0 HcmV?d00001 diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc index 66f82cb3beff9142909fb09eef3b937595cce845..a8f1591c92d8b748797f1582a6cd758517832474 100644 GIT binary patch delta 1094 zcmZ`&&rcIU6rR~`e{8qi(pK2k3T;t{h(bh6jlU8lXfaW7Jcu!+S=y~#+3wcaX`*;Q zy+AY?$lQ%L4<7O0(SN~6BH6^mM2~7v{{d&VMIppZb~EpL-}~P8US?ObZ}QP$B^n0o zIbj~Yv#}aY3uD6SLA00!vj7FBqD6%$#TbcsacHWm7$@;!f+Vt_Buo$T!ktf^;|F(Q zJVH_dwo$M0&~65+>>2NH$T77fqOb~(tOB!Zyu&7`QXmgd1VxcjN}*UmY$r&%l%76L zb5LE(UWKR&#Ydp`CJO$orfi(#RHnF(^m?Ntr~24A%gm&w&+Rr->fN&PUq*HZW1_9G zxLpeELEX$qPeEv-NL9h#QTkcqi`&^9PO21UBc|C9d;ivD(7uANo%if?nr*K}yQE%k}FR+Zgl9f@Ss@^5Gb#h!xwiT&O$T;b1O@;~Wi{Moo8`nuYT$Os)M0 zz7SxZJ_}zDEQq68)1Y6%PIr>!+I$h)Ho;>ozCq{ZOo3+>e4{O6L8@WfB1d$qIwZJK z)xdg%*!aFH)=9&31CBS%5OhULYv^lFWEmzr2Z6e-nWT=-Go@MjDl!R^^m}A3GT;Ww z<*HpNm+={TD|+c>vZgVSrZLi%s`16(0IPDt&RtV8D|NkbAA9SFXwBwGRU_J*=IA)Y zI9TFDh2wf-POnxC>$cNb68#!IET=h%?*PYWM(G>mH{X@D7O9u9-6A@6qn@Z;X*G0< zIIhgRIdnqFA9ei=yJZn?0(_K9{M%RPbi=N;O#LjLVF~`*IKz-WSF&)JzE+-UpTOv6 z;QI&???Gba)H=v+?1(lMAo!QmN9uCV%Dt6457iGa|5Jc)68`o9p?}jmZUz7pm!wD1 e^1y25<%73-*5ha2%{-LW!IjMbosad4+|M7z8{e(~ delta 804 zcmZuvL2DC17@gVOB%5q@vrXG<+R!GcB34V(QV6Zm;vtv{Vi>)6*oIAZZ6e8ToSlF@ zRlEw)FgFh(;y;Mq{SV$&3xWt9#HRG>!I@1|S{#^<`QCfq0wm#b{ zkKzM$PWS)^7twrGC2Ah*02JHSswtAH>O{{2Q>ZNsUKuVT3H$s#4wH8wbRDDGvyLMb9W(1A-x2{Jynj3 zbz#c%_FQ$ICfQe5)I|P@$f!rZjrEgNt;TO848w=A1)q{n(M#UUJdZhx&G|k&3%4y(XDbmc&VpRHP)Sc$Vpe8)FGE zJ%aTDD?3=PU=?6}4ptVdAJ~9{RRkLZHsoM^f_*Oxbi_gX1sw%eb+7@!_5j=KV1t5< z0ULL)p-Q4av?PzxbWo#~QmdMEv(c>c&*UNM=lA98y)Al$!sGX@k7$rzc3+xSp;Cha zyux|_nfwUw#EclgzqiOPIq;I_i$6)RLzz(eF=25L#`okD?+>ap&L6sOp7#N1DceS! zwW`g!rF>8|jO%SyV_}%kTr7%7VnY`^<#+8Jd-+{y09N=}%0pQYkpP&#BXf_ZFwQbC zaO&&F2m}oM2i1bUTjb{dlKO?Drbne|zT3Dz5$C9u( zJB5%!NCQ~1uGiYDy3WoZbsFJ0gg%5PwHA>?Nb_|$3yp(A5cmSkff;v;cwTG-p^Lyz zl>tckT1hn>0+mhz*;!n!i>`5uLCC@1hyVzWf0IgwZil_&d_0!o3qF+&@)h5GI>OWb zp_=WEx6|fUWR10(det;pV`bZ{8}`{?hTSmOVV3t&yS3I>XXjyn#S0TZA-|?Gdl5Tq zir)yQ_yhm!r06JDqsH=>DY5ou{f>)CFkmSAgr2$>$bjtk17q(xj;y1n_K{8DnqAJV zgfR$%{EtB1_Oo>K4otv1kmHNNwUSUBfIiu7)?20y@wc=((!ja|UV$K7C}?lMfw;^d1* zYfas|5l&FAXwhl@W#sxO9ys#=Snfv4w4}|3VV+RVTr>|dHlknc9F?PmJ9Q0v6Bm99 zVFBSCgbKnU!m}7Nhk*!_@Heo`N0(MQv&%nMM{Td>gjf5jO3QFQhh;2op1lERYdG3e zbBp$3S8xiu?df&Gq?PBno_Kgv*Q>2o+k{J3*UP-pySRK@d7xwE9k%Y)gG(q0q0C(| z4X(|R*jc*3ug13M68|w)>}K^fNEK^1tQN1^O#F+0?KbZ4D*rQnp&%^BLwI`qd^(YZ z{g)G~$Csz(?^x{$F1(8HES_G)K!gQ;DmiR_8E`nSh56gb(`(p?_SdQ(pTu1&F18X5 z#VaD(LKG_&d%W{4ITrNK@>9tAPi)+&#hV+`aMN?g?vszk4nX{d7Wkp=inw%xtCf$>i zp`aC+mVt=li}(j9$iP>M%9wlSdl~pb4u%wc<*WIN`9|>jeNNlkmL)6nle^FJeZIdw z-{-j>idTwSKBk2w{QGvj@M__bX0Tr_(uh>{_e*8@ESvPxx+Jw^!3L03JZwO) zL1a}As|YrPY}msF1sg#&>S0yEYRJa^W1ppPAwkEH?(rCh1*;?5>tQ2;?L#)tPZrjN(TLU{4v)9Bl-n_l=%&f=C|cmrq14*{9~5k&ZVSK5^>NL=jCU=9Yn8_<~n9|wNJ94;yk&2-KIeK_gU zjo0jY*=}w%>P`EwWv$igjn-Prwk!kwR7xuc*_k1c9q-3|e1O0p4?)YSHY%3I3q+*} z9w0bGFo@u-B1a^F0XxBr+rFGgoirFm4L%=N$tF+GIeqv<&DMs|u1W31ZoDJy^`)-M zsow(UcqE(~aK%&ZiUQ}t z$)lnfWZ`B+Q{*UfjruFKb^aK>P^Ci8_oN@wL+{63vuU^%UR|M9IKEnykI;k68yogz zAD8ik#}Mv4USw`8H*MZl*(@|8#nauCbsIEI!Mb#5l57aFa4RzEc4<-^iV_}*-A6?k z5uB3kO}pN*u%0OGm_fLA37}}>tOQ52vFU?M`Gd6a&Z(LulAuHnSpukt~-g4xPTC>%mq!)TdF_kxaKHJzY$&B4KHWcF-f|CSG1g8j|BPbKxk0{56Q#p(L1X+rJ`w>jP7%;v}OEhmCih8myhwkjY=g%G#6=_>k zn?T)0YlRXZSb${0#2fRe#1Xaxjl?;22KEx;-PBygt!K!`ooX&8KZ>|&(&`-im7E$Q zNkz#Wsy&s;VEnCA<=|#C!)>d5nY>j9?nkZ{$&g?P^8NW8(Moz;5!=B;mxP#$;^E|>$ UKVwX}Ar0M-lJIl?rc9~)3*o$5y#N3J diff --git a/app/document_processor.py b/app/document_processor.py index 0bec6f9..bef1f89 100644 --- a/app/document_processor.py +++ b/app/document_processor.py @@ -14,10 +14,11 @@ chunker = ParagraphProcessor(max_chunk_size=1000, overlap=200) kw_processor = KeywordProcessor() +import docs_parser + async def process_document(document_id: str, file_path: str, metadata: dict = None): try: - with open(file_path, 'r', encoding='utf-8') as f: - raw_text = f.read() + raw_text, _ = docs_parser.extract_text(file_path) except Exception as e: logger.error(f"Failed to read file {file_path}: {e}") return diff --git a/app/documents.py b/app/documents.py new file mode 100644 index 0000000..c96d3bf --- /dev/null +++ b/app/documents.py @@ -0,0 +1,209 @@ +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, BackgroundTasks +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy import or_ +import uuid +import os +import shutil + +from app.database import get_db +from app.models import User, Document, Chunk +from app.schemas import DocumentResponse, DocumentUploadResponse, DocumentUpdateRequest, ChunkResponse +from app.auth import get_current_user +from app.document_processor import process_document +from app.qdrant_client import delete_chunks_by_document + +router = APIRouter(prefix="/documents", tags=["documents"]) + +UPLOAD_DIR = "uploads/raw" +os.makedirs(UPLOAD_DIR, exist_ok=True) + + +@router.post("/upload", response_model=DocumentUploadResponse) +async def upload_document( + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + title: str | None = Form(None), + author: str | None = Form(None), + description: str | None = Form(None), + is_available_to: str | None = Form(None), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + document_id = uuid.uuid4() + + # Обработка is_available_to (парсинг из строки, разделенной запятыми) + available_users = [] + if is_available_to: + for uid in is_available_to.split(","): + uid = uid.strip() + if uid: + try: + uuid.UUID(uid) + available_users.append(uid) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid UUID format: {uid}") + + # Определение расширения + filename = file.filename or "" + _, ext = os.path.splitext(filename) + extension = ext.lstrip(".").lower() if ext else None + + # Сохранение файла на диск + file_path = os.path.join(UPLOAD_DIR, f"{document_id}.{extension}" if extension else str(document_id)) + try: + with open(file_path, "wb") as f: + content = await file.read() + f.write(content) + size_bytes = len(content) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}") + + # Создание записи в БД + doc_title = title or filename + document = Document( + id=document_id, + title=doc_title, + author=author, + uploader_id=current_user.id, + extension=extension, + size_bytes=size_bytes, + description=description, + file_path=file_path, + is_available_to=available_users if available_users else None + ) + + db.add(document) + await db.commit() + + # Запуск фоновой задачи для парсинга и векторизации + background_tasks.add_task(process_document, document_id=str(document_id), file_path=file_path) + + return DocumentUploadResponse(document_id=document_id, status="processing") + + +@router.get("", response_model=list[DocumentResponse]) +async def list_documents( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + query = select(Document) + if current_user.role != "admin": + # Обычный пользователь видит свои документы и те, что ему расшарены + query = query.where( + or_( + Document.uploader_id == current_user.id, + Document.is_available_to.has_key(str(current_user.id)) + ) + ) + + result = await db.execute(query) + documents = result.scalars().all() + return documents + + +@router.get("/{doc_id}", response_model=DocumentResponse) +async def get_document( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if current_user.role != "admin" and document.uploader_id != current_user.id: + if not document.is_available_to or str(current_user.id) not in document.is_available_to: + raise HTTPException(status_code=403, detail="Access denied") + + return document + + +@router.get("/{doc_id}/chunks", response_model=list[ChunkResponse]) +async def get_document_chunks( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + # Сначала проверяем доступ к самому документу + doc_result = await db.execute(select(Document).where(Document.id == doc_id)) + document = doc_result.scalar_one_or_none() + + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if current_user.role != "admin" and document.uploader_id != current_user.id: + if not document.is_available_to or str(current_user.id) not in document.is_available_to: + raise HTTPException(status_code=403, detail="Access denied") + + # Загружаем чанки + chunk_result = await db.execute( + select(Chunk).where(Chunk.document_id == doc_id).order_by(Chunk.chunk_index) + ) + return chunk_result.scalars().all() + + +@router.put("/{doc_id}", response_model=DocumentResponse) +async def update_document( + doc_id: uuid.UUID, + request: DocumentUpdateRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if current_user.role != "admin" and document.uploader_id != current_user.id: + raise HTTPException(status_code=403, detail="Only the owner or an admin can update this document") + + if request.title is not None: + document.title = request.title + if request.author is not None: + document.author = request.author + if request.description is not None: + document.description = request.description + if request.is_available_to is not None: + # Для обновления передаём список как есть. Если список пустой, сохраняем None. + document.is_available_to = [str(u) for u in request.is_available_to] if request.is_available_to else None + + await db.commit() + await db.refresh(document) + return document + + +@router.delete("/{doc_id}") +async def delete_document( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if current_user.role != "admin" and document.uploader_id != current_user.id: + raise HTTPException(status_code=403, detail="Only the owner or an admin can delete this document") + + # Удаление файла с диска + if document.file_path and os.path.exists(document.file_path): + try: + os.remove(document.file_path) + except Exception as e: + # Логируем, но продолжаем удаление из БД + print(f"Failed to delete file {document.file_path}: {e}") + + # Удаление векторов из Qdrant + delete_chunks_by_document(str(doc_id)) + + # Удаление из БД (связанные чанки удалятся каскадно) + await db.delete(document) + await db.commit() + + return {"status": "deleted"} diff --git a/app/keyword_extraction/__pycache__/__init__.cpython-314.pyc b/app/keyword_extraction/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4453209776ef0e3d64fc7480fa19e360ebcf0f74 GIT binary patch literal 821 zcmYjPU2oGc6m`C$dM*?_l2f|r6w21pcDFKIiz__qH}Y1o0}pbNs_b=$9U}0b>&vNANpG2~z0D zO03w5Y-R82+>Seuqa2fW;%?MaJ(D}J8}(J+1W6TC`=rc# zC?*p<_`E2W{%NkJzHC`Cn$;D{Q=)QS2Hs&^Ra_{V`HX1236)R%q?R%#rQzEzCoGrK zz#Ci5jdf#7JKzRZZocM>@dfFdO9d|Uri?0@D%|p8nHLqKurfLzf=mRdrU!zjR7z8S zbxFKFx4lNczc?=!XWa+EUhwe!xF_@liH-(GLsvsjg9|Xe(zqCBJjJ*fEW3gwRc2Rn zD_uFqtLAWF_?pqB?Zq*mX-1>DUbW_N)w(yr&9zozHLtxiJL^izTLT$wu^C`|LWKl& zFbQr6V;Fx&7qEt!kKvRTG%Nwil$Vl<6S(QR2#Xms=@^cV?DdY{yow zH0R#A=iJx1=Xv*UtMC&jzl^uPF}t0R@A1bjE~~L|1R8TBN^~+tqFg_B)}cGjI(6q+ zm+orDK1aV>ch`^>5_Q&*sOzA^9I1Pvjy4i@&x-yNsw}HPboK@}Cuzge+-1l~U?uQRdZgUcD?)Sxl#s$^;KsKqWFtLR54EWV|IwE`VEOO|e zHHYr8a4v>(m*C1QoQL7cN^stb?r1ri)yuFx#L8S@USHI&`(s28L<7L93Zt%wRVNFa zQt9l$0)I*wxP?ole2H_gfBdkx;d8>baR|UsZjQvrn4|YMge0K1iTsM2oZdsj{sNvR-I?PcY%&FnJ9dsK&Bglp%XC=Y9 zsXdQl!WKdttu+9cBWoTKs(tV3wX5&Ey&7y=3by5fyYG5;Q!o@J2kdj=;&`r&D1avl z7FY>Oyim-rO^U^=$5H$jaROId)C&_EW#&GprW_|!wKxNq>z?lZv_BL z@EBt#-ZKpWs76c#0K0IXd;dBC@Nv%?RyV6y^?A+b+7~Cks`|43*4r!3pJx>2@$u5_ zV(C@+#vSOHBcnE3$Y_CT2x+#I7&&NB+I8CT)MGh&j9sUlF~?BZmL=df<}6USM)9Ip ziMj0FQDokhz8=(cZg$LnLX_hf8MLUx5Hl&{FG>2eBq_W;ty;1Qb~wt+0Z6K4xP`cy zQS{E@;esW^{;&s(hNM#utWIkH80Bz#64k&0fPx!NSyBv_3|2ktru9gKVri6_$^cSe zHVznN)?ykmxub*Uc~b|r$L;M47KIUhoEAJpS_N}y!vN;UkL9GQcDeSM)!JVy)&Anv zM6UL9uJTN_>|SNfBYN*wHmp{5ELC1{LduM^=>yFbHMZq)LH_rfN8tY8b9dsw#&aPtph;*4A19Fpa2oSFaNQ zAD&pl>P8!@Znm?|pHYJUzyzQK0L}6!i?=EIAo4aLc_d{)(9_N_l6J+MA#(a8A!&D% zOM7Cj(Sl;d9JWaJR)S00jV^7Zz1Wt-+`X>?224T|xHud=#&x2Buu9rB7k`@!?r~7$goXx& zX_&2JfuttbdotP{r(H!v=wh`Q08FnMx>xIuE!7{p?a9@j$yVHh|AymB4aaZS90c=v$c6Gj-fOE~}#cWF_xK{)70MO5TS` z4&Y|u^XI;(`a-(BW95aHRt^oWVdpnTj;%SdK6n1B%+f2bF1;q)eMMM#`Sq1^;yNN8 zwgM<0V8_Zgvamgjg>1cCJeK|AVgJD)3E+S%^(p25Y@4hIfu|gZz)m8%a}Z&azJT@ z@CR(ON9=`23xX5KD@#L}VqNRxr@)?Krm0U#d`i`HUQ0{y@)fw=W*B@f@H^i$TSE=3vUow*HL_qkhMN&m{XmDg|sFBbL6w0 zPv5+?{qOoeW^c~^?P{*)mM-gzd5`>k;JGnCn;fw;hK`tx>1hMN*OhlR6>f0B3uX? ztaQ?E)&j=p7${`qgn;T}1d3)b0#_iJG-w9W>4R1YF_3KukvS>_%nArIsWBb(@=KsUPIHu#azSw?790p zc4Y@PJcRF@58T_;vfTRoYHR;eYyX}7xz<>2*9i1B?pk;&*SLS)3v0HtFAlD@_bs*e z-Fhq6K9J*Iny<+7&wS?m^jxmHe}(U#ufPDQ|1K=yLEZBad^3LIA72IDfnUVlfc*Y2iM)C zy&GoO-ta;2dT`OR9NLp_-ZNjG4^@8Rdm?kzLQOo#TkF;yvFr-*x4a zMAoQk^f<8be3+Z)VdeW{;c~+(2#T1(_ZYt*q|`)4#x}gvyqOVY3lkKC z@g&uB8Ny>l1$3Dpz*7*?Y4DhnjHe`hQcbXIk+#7aOw=hpI~X(4V*1A|9gi0*RVu=4b?8u2JFvRANlz_VGGiasQ9PU_##LE1W1A(5RRV`L=u$vp>I>9MAD>0(liKxl0^}PfTIK(1faWv zB{P|ZO*&J$X~s^&E7k5y z`*sfpfDGf^*?Iiwn-1$s~|s0THI2I?rcYDhb%6Lf=mK~Ku`P|=`4Fp#`@ zsCckMC~=@_iK9_40vFAYY0xZ~2TO%g!c{wD8MF%4!7`!Df!sQwyc%6Fj=5y<^6A+A zd|uAE^_`TIcYKy!MfqNCgHV2JfV`DhUcpY} zwOp_pP`e^ehqY{nH5Tl>cIY!FRkDu{<*!<(Qdx7?&fL;{Grr0GUfa>ZC-?$!I(x zu$)sVs)j?nAkv}r$?@?uel&y}FF5sHB*bdLx=OaN6C z1;X^PLEsev$)X(asyGq4#QA3m2nIrtAj^q154Up*bh1dDdU-w?7-9Wn(eTq zc3{>dM4w?Jp;?yW+hXy^Gs3Jn5anV~&fk`sf@ik2FM2f+j{1Y33x;E_#6uh#WJZTQ zrT;ZBYjb^=V3s8?;^Tn;%k$ImaQL~|U7+1? zkcmWv97%qpBg||wVS$@N5nk|z!)(ySoZ+!#(@?tT4?M%nu)^hNkaq(CC7J(=N z#>}(avmkMNrfufAF)sQv8xZ)mU^KwD@~oc=TyFEnVr|c`&tZG_<-fMJf_cf+QO*^6 zPBi#@{s=6wUtoPcv34hAxe7_}PRv(8^k@z}L<)L%fr3T*Z0fO* znQG~uSKZN8a&{otVj9IzfydjNg6NY>!>zyu;#NWo$0o!IHw`f^;}U`&-Jxi~GG{|! zzKyO&*VSAZG>Iy_U5eVUA8($QSdoVP&=(Ok&$8UbD9?&&juqlu1IEbNa21>uC~JR2XMEyECz)qk9#o9`nWzgoXR(pwfO zU2omDp~MaAJ$>2ClV_6N^T|nH(m$P;oL=)@T6^;HeL^E?kp`}E!0rhMXW0A&Narv2 zSNLsn*u9{J&>$6~9mq@9B5&>pi>W8^EJzrW9T>R+QlM446<%r#o6EK!8(Ak-285W# zs~H1R+#Y1QsLh39+DW|5Bos)Y+Xu(*3>DK)8s*k0g%^#%fw?_QQ7r{kvWsCt{nS7| zLecU(he?Y}IBt^<+wt-i;=oBQ+jwO+DZ&e^Css6WoA@#Gkl8>=p_sv~7{k`HEqD#` z7|{x^VG0%6T&#Fv{V>p9(%=tuLd1D#!#jNtA+N!0*+O07Epb~5Xoe72d(7xHn9u;2 zQOx93xK)SF0uPX*22EnSB*ANvWyoy}ur@X5G<<9$rFnhi>86y2UIe)$s(mmH9Wy%6 zh(d>UA+K>8m#-i+i49yDtK3`RwRI@n_@vSPF0?M&i9Cmx*=wGn4*f$;zJCE-ne($4 zuaGHQ3ci4;_Lc%0jFpa;!vTYJMK(!qsaw-hD$fCQSvq0@3MHWc%cwy>q1{U1)sEq> zcv~2+@RoMuz9R?NAJ#I~8{`J1TpNm6yjobbI)tvmMjW%oO1)OEm9~V5lfDk7>v@hltz$1=`6f+Vk7Le7}qWK?|2|*RRf-U4o2`k+5L1K!_Xqkce`?Io&x?nR>7n9 zVd@)*s_srU9bYnM?98{$e*NqcyWG1%eLt`iPT4zF;z|3lhe}jq|Hl4Q)A9f0wb1WN zwH?%TeFt@^gM`iFm!J~DqvNyt^ymBfPn>!D0y7W_!yyO}*ac>OGR*GgB%v`9I(EFboB@P@x>g^2i-RG~@3p-;V! zm+(4dMCuxk^Ri=eEbO1T81#3Geg7vtX1QDi&O{w^Xr*?g_lFJSl)kPImEc{98jc0` zf@ep&vBo}BL6mqs@skhn;fphgm3;O_KKnY!Ccl#WVwQkpB?6Lt{X@)8$|k?~{(nR0 z#z)kNek zMno137yUt*mKUjzSR}y;2?Xn>LmV#wUQq{lBAYF`?9ZWLk-93XrU6{SM6of7mUtv| zCC+a7#zeI=vS^Vev89p2mlqry`-nRP5swd4aIu@3nC+pbr$_qG?VtD|i|wD?Xo)j= z0xaXBIRu8KJGgFZThXR$hv$p$mR0A>3bEG^M(g%R93gNCuMD(*NGKZ<_-UDE?>B^*pRMjO;)w8P-|8D)8_X1-n(U0 z-|Se7|K*Wo>ZR`G-nHu1beU^`%IvCN>RqP(I)M%G; z+5FPff<9xhuUhslKfUT2T({OhgMH7k;-x1RbQw!cg1NkExhz#X8WxJ)sccy2&9rvi zu&%Z~_Tm?ouB0oRYZcBnJ8oXNec-L$e>#$B+Phe^ZnNLlpsL1)dSt7I&M*yI^?%^q z?mO)FPOi}3J$1wU{m~mwuC;WpHXYx@4G`J77gR91)wa0*YlrXEFyHA|ihujaT21SU zVy&iqpm5aSEKAW-Gmn<(CzD{Q3dctZQY%;?p`iip_fb2)`Jg5 zC{*5dUx`XPHarw^)F!mm@7SHskIWA&^kq^%hP1YSv)jgK7ci-UC_LB+wNr~xu z#`5Q{-Zhmi(u>oJXP2H@p;rRSpSw|;C_i?`^cZH?M?KIYM`NP#P`a@@+1Q{;J) z;HK(#y_yspD^xCX!rg)ki+g5KEOA@>-vm9E(_x@gj0P2gj0b#KobbJ zePeLKkkHf&34v>QpvJZ0rr!$FJ_l5MKSTtk^-%N(pmdU44{S)8*T%D%b!W?llC-TK zIPzN8Ez2$TX4Tr!;k5%J_oaGCIe0Ub1NJgNIfl&$oaS?AQm#jMU&G-N9Ae;wqOq(G zB-KjsM5$LPqpf=RG8nFvV=8<#Jek92Ue!)GERVNjA0bz9i( zJ;u}?wF#Zk#1$?SR}8@B7Snh%JE0`GXKuDIOTZ`KU!RK;uUIPAgNK~+VqF-<`qZ9yOK_a(ID0zVQksLq4NY}=*;Mc@utBluM;5oj}a ze=t__>$ISm+z2tfM?aSHxAU}9lq`b-0mTkp_+*_-J=P%S^68|kB;-*}S!JXvr?DbW z5%}qFh7d%G4S!qk37aBBqOj!+@0n_iKL9@Jqzk3Y=HZ1k%3b3IiR^qXEO7qyz zQ{pN0Y`GaIuztW_QhOmboXkD!cuL2xv$ZYAUmw0XcB|ZU4rf_mJ>|xNK4aKT*%mxj zw>oEoWu7v(#zXKK19&Nax8unn;}~s6{fgl(=|Lo}aKH``(}WPFhZbtMNotYG!+#=| zCb4%!0z90+Y->=g9HiV~S3Fnh$?b}91w6-SV&%un>$7S85yq*VW%{LS?5)@%PVVLL zjXj`7$+yLj?6biOt{b{=Jvi!xr~qGZC!s9%e0nb}5*D5*fM;1)@hawuoo9d*{u|T} zgXD1WaQLvw0jA9dQr5!MI5AmWIT>0>GKI|F$rU}3U?0S)f zTT{5C0Czt&6B3m(p)3Trh#6rwfy0-9+%%3@9N}G5ba3oU^jX#iKQ0V>0S<2QphaE( zwE!E#PhrlA9pQl)vnjz7$~|fUq9C*4@g*-jCW%H#Og89~>bVf^RD6*78PI$jw8be(i@lDMVl@2q9&W;c%b719K~^uf=hQcC{iqW6W9>}FpLBMDt80` zWWijY&8&!f^Ab4z@B{&V+^#SGe#TOfwsa*eT`R$xY|1j6Fbr>~j_5|HcMYu@DrNEE zyR{8V=U(x>>{~gT0(Y?X_(BPIh~-u3vfaNd+r6y%8~t_tcL5`G%@6!r`yud@9W3=~ z({fp+w&@l7%l20s50r>$%yb<|c8!tj-7{ws=cZC;e2I&JLr_p0sl!}pAqw6QK}tRwXs3Z1#~Zo|=Z!@w^a25x>K z)iALzlCjpNtsO~g$4c+2^+0A`o@I8@lI{)SBn?aAhM-0 zQQNs{>AXQFdd3r;r`CGT-#K(X(|#}k&_kWFwt<^p=Yln?cP5<2(#{h}=ZTcFZ@uNn z%|7t*>rYX@ZbOS|m~?eZvbrT*J)Eo_&VbWDzNu8yl*|u9qm@}wFO?-MyAy`pk2Y9} z!iM5nzLu&z^k*gi^??>IT+5_2JPTSC7)f;Lhj!TMh%&2kPWP#QrcARBqt^l?#Z>`3u}*0-6tdxhyl2g3u?&0 z6mP&Q6t7z^|6G?K>XaxmRZ9YDl=uZVUbD%YyFe)M1 zk@WNpD2iq9h7PaVp@0wW%em0SxWMwz$6WBr9$>>^pAWnxe0@cH9qF!$gM~*V!cc-g z0nwvB0xhho4n0)iC+`o)llMa{e&PIpJbXTMD+q@EKtk6K`xMwN9};twjFo7GFm={) z-^K%+he$e$;T0bio9Ez2LvX}DhrNXY(2s3P^60S5au|J!#bI^^{xuMqC65BqeWb16 zT=(VeD9Vv{a_$SbpE>J7fNt&wP{ex`Z-W@NRf@W=P*AE(Eu!@ABFB5EK8fnzL;K%Fd*4NmZK$g$(>p5T zd#dsGR0H>_nS{!=p(vr98Li<3-SfH^N}exCR5(|)d$&~VUDY;c9Q6rp%|;a}-#a(@ z&~#Qod8tiAl1Cd(WT<$6^t5wbTbwA_zp8EDP-u0|jKTE6$RCd^?q0U88C-IydD*jO zXx~<kk;}5DJj#sN|(?dTvZ*)lvT}*W{Qko?Ay>l{YU&$prP-T_i4~A<)ITgbnB=B z@^9#jeJbS}mA&edwTjygTy?uv(}(EWdx}pUrEhnsp!xP;Qh$_&`Zp1-c~hkX$~QHZ MKD+YG8Vch70i{?YX8-^I literal 0 HcmV?d00001 diff --git a/app/keyword_extraction/__pycache__/keyword_processor.cpython-314.pyc b/app/keyword_extraction/__pycache__/keyword_processor.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fb52c0abb79aaea2da3ede44cf1ef5ef648e516 GIT binary patch literal 8792 zcmbtaTWlLwdOpLOC|*QTluXK&tdXgUX-kwXDY306zQnRBMX@4}opF^5k$yG5e{Z4qOELP6420rtUpD7r>lfGlN;9xI4}?Mr|@NUIc1 zpW6RFGegpn*LKkZ@XVPr=lY-TzaAc}b2|v6S7PmN|K~13euFPYvgn1accHLAIHHhg z!ZBlv!obrsW*Rjs=2omTk6A{oignbc*hXuV8mMD9%b0z%R;e9zC=U8=9dnMl6xXO* zagWw1b+pbl<{9-W-d55`+6Y$@Ae?=`q|K`MI3`5Gwev%caq+!tX(`TTq;yP_}+JCl}J@wy^KV@e{OVxy@zJCjH%q6Ez%j)^d%)|^gfQ&15$tCrUivZ7iq zWHU)oHJ=xiut^Qfii(iQreaDqibDvQr~<81sxu)AvXai=+|bvT6zArk#k?p;Sy)9h zrMB#-5@P9OG99A>sa_ah1_#Db=_ri4D=sQxOc9dN)ND37D++RA4i*zLXpT>_{<8JA zP>2HMFA;?S>YJwBig_1F!x<pefYUaP_p# zPU~v14tm;YorBgnX}em^N9$a)&aJm|aDG}_M{7NLt&{UAE}kfE&IISwpnCOVn$%7y zEzKX(i*QO;wIh@VtIn#MHONQF)m?RN6EPhHloD75n%KAMp|C(MdQ-F-GmeDFEoR~t zoH9&tS(NZZEwY%LQQdllFqe*tNf`&}>{Q?Cy^@|2dsC35V(FADO78+>vU9z2@4;%{ zhNF>tnBPRYcf9%M*4``Rs=d@*c>(Um%Z&#TwYe>{e_8AD_kOn-qZWkDVj#3VyDw--p5i znKE`vcxFH^Phy*GX+VGHn4{1!?m{X|S&c@Oz9AR~XknV)_nN_}2JYCz44Y88WR}x7 zH=>%PY}hP0VHDMxOkWcvsTRs%lY|(Pc0;BkMkCHH1FE8sm=RJU%Au(Ap4zp8Oiyk| z3AP_bT_FGAY+LFoIuEQl4;7q;iq5{}_^NYo-G68~Q1l#B3 z4>~dyo=lce0R!rZFZa$^r;zbzx_H6RGhKR(#$Dql37JA9&{XLQ9QY9-vp||=GR_gg zn@%C18@7KQKa-5}VE{tf5puc}eLxHp(VJE3xSrmL5YSY@@i8 z+F^9512bH)8Pr*P7ZW4}aQL^mniM`wITPqJJPaTxx1rYZ@pt z4J@B4HXX^my6)e*IKSpUwCX?f#rYNA=ln|aAKof?gCBctdX|`b-hG?3q&~3WAl_ik zvlH^TZy-6@8iMR;kk=s2nGxhD6szIiYlZ``Xh63pAj4I=p?nR*BiJeI0sYTFd9vWMqguwjGeau zv-SY9)?QpZ2D{PO0~mw1PkM~@l_y_ojM+-~nn_>fi&3g>G2W{)Pz`6QK2KD4^PT6Z z*;JPA71hi}fvuv+pyapysc~A&VsjdNxJ7fuvtU=dys)!7v;O2P~INe z$;LY5$yPV4@HyF-RIx2p$Mg&+!%EFC%(FQa&rIG28RgU(Ys)5q>@(n0czgw%3XePe zv~nkxBKN@0wKW3;Fu|BfRK?qp!3j?@S~6zd^g9ffiefVJD5FgCLz9x2iArJ_)|W-F z+e8*HiB6A_W&xt?{<3iIW>aX^CBY{NM3 z;8v;>D(F&u;&sr(Kx$+2x=m!E=8*IfOfDg{RLkXbI{9n}z6@<;Bm#vXYi-&jklh08 z!Q;yiecbQu+c3lH{o|*<8om>{J$Co_#3oi~RKV$sRJNB!Ks`iF|1VftrpmSwZ!yI3 zwD~BQ6`Cc))qp*1E?Y;y2aj3ZmZvHoomr+eiyaQF8O8!-aSD`$wQ6STbB$s;`m3^X z0gzuj^R!;afLgjDf+ELoCf@3T2t}W{-;_1Ag1<?OO@?=k>ds(!gJPf< zD|ECOwbKNx#W)!KumMx)~xKA#&ctS!vA+l*(mw&4Ca9#98gnob`z}8RauO`6Yj7J;}j(U{xpaU(*{aOnFKX?5rpATE#*@INw4mEF&rH*L4l z^^*nHa+03k}1b(R541pU{N6i+uB|x2#Ktzu2>K?zVmH z@YpBketzzk*8i66(BMZ6oy=c2IvFacA0r4dIVH$zP$-iFT)4-{VZh5^N}U)Gwks0@ z7E(DTh51?~4O;0+8UrsE<^y1OXQ zXrbe1v172<_R`8bt8FJAH22)Fulsl1xc>3`H{UM?x{6?xox11mS?>Ag<6j*wo<4uC z?|f-bYd&;zCAi|a-Ii|~&hHui_lDt3D`|;<73SN!<_#6Rp`y1VXI*by9}F%sA0A&!ePLcc^SNVn?_kN_mv{ATz@i#7W!%(UXojONHlHarodFll zQ-9-y59Sw7efa*;0lnA0!BXI*y!WMto?zY{+^Me7SD@;88VW-UD5|H1ur$9-Q`ekT zSJ&EADDfpw|il%%&-EWFO7Cu|mjJNU1gNiqhpY#P?`a!$vi@YEGmS=`1$I zJB4?_mW{fG>p=Lgv>c>^C`Lg)(~_dE7)nDKdZNPCW*y(1Pb zTr9jfoxd=BcS5*3_QOqVpwXBl()ui4^Q@15avOVPH1L!C+ZGS6 z7wZibm8qL*G`<^VLdL^efx2J!za+6_i?Ujnvb_>y=z*6`ntL3bs;q?Md!dx2Y<^Xp+Jy+h#BmV%)$@T$$8%oPQU)zYp0iP~T+tE!DmomXlBXtn?bWmIbh?)4Pa1|1*_DxHQp%`F?kuQnO%5Qu|X6N)$| ztHG+_gm~4uB)mXUTP1NW{jMm$Pf9{u75%=d9&&iswg^z=#ML@5W`r2r93|p#Um|PU zsKefsy{WG1(bW2~+aA}akyTsfiWHSaaMuwORacC0LHnYaG#>4e(6B;f)mJq#y;+o1 zMo=yE?#HJGpDL?UeHvY=7fdPOst`qt%OFuO4p0tGp`ouO4OhF3Q`!k0_2B;jX;hK^ zD*zVECL4ipQ{C706Pr+qtejhII+3d@1v=IOgN4B0%0T`cpMUeMyXW4@$FJrCgT+8H z=c?E$rM6Jswf8|=&rSCY+j{${+nvSsvAnmf6b!EgdkewdV({<{D_T_9KO0=~{rQPf zL+4sUPobeF-+S`cp?^AqxUwsM@?8D|m*=MPmoDdH zGx@36{Opzd-o(A&+na8(FS1cb+e@7u-TOD}q_KTzWa*7!!$F9xd;Ix5-S<4*B`;Wcw7K%>`K7D(ynSH)HFksf z*V2kJv>g1&+1yCU)0l6Gta>8L9r?lWUys}!q}F5iVW7`aXVDvmPJ6q*vy+xkJ~Xfv z8Z3kci=m32!$g)X*RlOkxV6Q9s6-Ban@R>+pg!6`jNDMTdabve_h-;o|X~Y&5BV@d(09 zLXwqaA|Py39G5%Je~_xry4-8!0U&{5#d&DOs7Pkj{iGyEYsWiqh;KOu0B za!^6)D&>%k%7QZTln3SHsR%0IDIZmisDi3WOrsc8kK_b%Ml?aqh&HGl(FJuQ`kX(V#+~x$c4CIsY6yn*$knJx(n_FFLTp#_qLCn zV*;KaGrQ;x1en_k!5hrd@?vmdalw0i*}cdFmi>Mo8)V#GOt<9r1{Y?TInUg(e{o^f z9bE8v?YfIL8Lyx7%)24u+`??ors9?77Xm?EH3i8$ykgQ5@)g)tcNSv0$=`b#+Au%wKP$ErYw2pETXCQI87zefJ92F$i zNMcRNSUri$BXRi{7o;7PB(8wOnUm=ZB(9Le6_IqgB&L|eSV)Y~Q4%ycsbHQ%4$Fw) zi!Ml3N^+24szcq<_>5IX)|X_U53Hn%v$%SbE11fMFb?|uE%16}_b4ZIRo?wO6g3C2 zR_e0shRh+Gls!+uA3o*yY$}DsN!Koy{&L`Zg+oDnC8l+#h_41;>Bs?J<5T{mI?4?SdR;fZgwGf*A=|MdIC3mi*tOrq;D?YEf2fs_DfzH zJxkX-b91n@TuZ(=&mvFHx@T{AT=S3@*1l;u;E^uF0H3?$zT@)3Vqb#wAMyylXKyTf zZwkK+LO5BIP0gyYwTeKHWpU(q)%>E*4SVmJ&$q}c7Q8{&c52*d5NU9^$fm~~>v9FK zpVjyHsg@hQB~ObNJnyW}8}P8VVCye0wJhC*{qrJh$w14TZ#K{z@VMF88!c|Xza=>+ zF3+7H>z>8^or(g|V)x(WX_w3G_49H%L>6dH;1l4zbB}sL^~+S} zWsii%>opDc75ArJt@-A2>$%%CU3&`h_HP@S_EbVTHFfOxUXBpfP&M^?S|O~Xn3_Gk z5T>cx#yx`&&ZVmB_KZT5q7m)EDCYUYr%;X0&2x`jaAkc2ZuI7&~JSLmkNgyv-5goa%;TaE`{k zFq0>~amdmfU<}s88uTZ=A9mNsKi{* zDrkY=B&|@VbIz2D?;Uyv-IW{}$Y)H)!!MKgs|>lQVbnB|=LfFo)I&R|30w{boFi9K zRw_57Wey~SpAcK|bZolL7D(YujTZS;Af_k3QsYN{6^ieiveZ0$3g^Kon`jo_rJv#q zny8@wMcsz+X8*_Z)Co2z z0;!=GY9>XM)zn-!_Rp`Atd^W9SZxkXn?h_Ati{x7b7-ej>Gd|1eZ5I=PdU`-*PYP~ z%Fdq_LJP75>ZGEaT96%RN&8bRF;3~AB_?T09D1lTwZ8Dz`>!QQ{Y?k!w-toyb)y0= z0eYSZwUT5M;h0?)GD`qWfbn?U*A_iI9* zW0SGCb3!E-klirA3;?~?zysJ{T+nt|er*e%fF2V~~I!MX#U<_o|v&tGUh-`;$yEtFY+ z*bH}oRD_ivf;2$?uX)&@z#1VU2Ux?oK?n9(MQRQeCu2x!-M8GpfMHaa#*o$4#4HD} z$=Gz#>@+hPf&xsyW@nLUicH&9#^Nq!VR=(*33|9hSv`1s0rrC+OqXATm@HB`o=ziQ zAIq!!EU;8w?hEjmCHGCw+yWb5kuR`EF|l^=&a8*PT^5NspO>8AbfJ7svd`=hOkheT z2)M4@CBT=@7qg^*Lx3+3K<1k)PRlqV{hVw$qI zYC1Q((VD3hWlY2DTHB-6P7Id(Twk_J=SA`xW3=tDn$qe&tN)aK_56Crwx$(-=SED& zHd-Rb&P8(jW14|Id5*Sgx1?#^6)SmmW#H3enAzMMqgx&W7HHd2#;OLf+=@hT*^2Vh z^qx9L+x7Pt+7`O;oqVtGl`C?w#tMmvdK@iGt#IWd~Q;5i97}ZR`A#y48WW z*~Xb|>$RM@CDCyle;?({N7n~9^ReB|vp*<^b`D}>6K8H(pW@7IKdWq59Zr}naWiD8 z4Nq@96EWBBn8)`O6w|nGplVJ&Qb5h`Jno`QC66g33;>$)OR4B~BlCjXv_;UJhh^`QTxc(-cLjz;2(r@0QNEm0sk#ZfcGjE**tp|JYFM; znrsvPRI{GoGV8U;S;Q}FC3q07!f-WsNxV!}Hj8|jtpm@6Z*VUY)Oto${)htaEA4yu zc(sN+erzTW?nN(o$s*gHP2oQHfiwtsD!so7u_*OSK*T{ek>tA4`00ERN%~cgODTOP zgeOu6GX2gePh(LLYoy9iA5uHE@qn9lIMp4=RuPPI$C;BtM=*SVj+Ck2KzjkTrZBEU z=9B^7l{@9&E1Vj@o)QG_heuaLQA7Dfd7J;HU65jZD%AT0|yL zDw4|sPRKVsk*$r+3@P*kxT>h>FNy1P0{7(sCzj~Xo&I0lzeiEmVQpHealpb;xqjUg z?bnZY2{?5+Jb_#Kfb0Ga%oObGjCBhjA!FTY$hy@!Rj_V#PCfW~r_MneK!R!0ifg8l zBJ)E{Ac;8OrVFQS9m#nDX(zoyfD32Tlq;#pn5rX3QirPMSD8Uj>|}4BQ|>fnZ6T1( zLtB{Frpjy?An%lX0x|%M1jqo!sdRrq`pPKHG=&sFKV<}3piGwq=sL%5JY&QW6+#*4 z{{ML$w){{@-`pJIPU*D(9|v!E2x<@M8D{e4g5M7yyCI}#0A#APG0n`-)O{JB2kI2i z2nAh1p9|EUptigiq8Y~FL8S`Gpl)EAPeVo{WK1GeMBfeXDjIs2cNO;bd0u7<=@mejgLNumIiU1UfAwcPx@- z-sry@^nu0*6kGws;k*(7DuJE65v#^=2|yB1_Oi&%ZFy`j=01zw0D6Qk=fD_V^94La zUO>R=r#rIWF>rg~flBlSTS9EX{ z9r210u3{uXmqhB0?K6RmCdyN07{!Xq5rynC$oJCo64 zeHdBG(Y4`cIoghelnBk-Xxz4*jOkA%Xp@*=o3^iiEKZ-=%G;*TMd?#eD*?)3Wt{2c z80b@+ITK^f?8z;L@&r@+b=_BVapoAu9NTD)Oio9bV^QYWRl~!o>ebvt!_kPT_F?bY zc<&_FI~kdp+3tNI-aE_n&hGSjICD1uwDHA~YYEp=*MwntOu>UiC0uI_ZK;Pjit6}y)5Fugv#KDy!Dys&xZ zK}+O{H}ae>;$!zHI3qYGheyAB2p<92Fg}LmBlxHsl|QDak}>(~Wz0i_>y4bbalLrU z5;H%SELXSQwQk!uzS+E485wy#GV@~OrZ@6DllOq`qG17#M&ISPOWMaCUe|W%b7q7*nD>DZp?J~&4SWhbIICR_;z?<{np0PhHI-Y zQaThf55Hbq_Rix%%3S`KQW(mGbP;RE#`MPM=Hyo1mLgI*5Hk0Y| zkptkwu^YJL+IiGn3){0 z6wqU)@U$350X`RVUsW_oBmgniFP}-pI#YURzgmJ!FCO6X3WqgQ&n%G%luq?TN=KY- zMMp9S=s+da(@AndnPQtjlbFKG4z*L}$dMvZC?iAbXs7%+@TWc_CBhvY%c(*ZYKB+iguD;Y09-WjM!D`{B@UqUZ*(vfg#0YwAn_3NO1)aD#; z)Bkz8MnrT3X@AF=lNICZ)1&CrWKBV*Q;Y*2L+$UN>;fDnPNX!x>B=WkCjsfW>7sO7 z{02$S$H@2@QWDC!10x}6+YIhZ;+zHtEuG)1GI3O4L?9LR)}Kp!qu(T183t!AjAE`6 z(5A!aG&xLw;8krBbUh9d2KJ8V#J`X%3CcL6r&;6?r%@a$e;(wj5!;-8PXDtb^7 zdXfsH`cBX?%s>|T!I>wP3erQ)2c;Nyk+Tr7;B?iLGcP?>{(NV?m?SAR{RZUAUV8pj zDr3Ymex=(s{>9|@WnUQ}5g&9jN5z<#W-;VU&4^gC`1BV@*3P8^Yo$Q4YcygGm}#?g z*GN`F;S8c%;v8@o4ztxN{!Tw95J(?zB5g8JVkljk^fb*;>{L4}PK~2PQd8-#rzS8q zC?K4zJXDu7d0>#aflQH9+A|;eKjVGjC@*7pnJ!e8%!?Al z6v=)*X^PRqu;bA0kQ$?Vn4XJo|33V@7*gQL0Iiz@%a9-gig6zk(lX?Lx-no29nlR6 z$HO4%h)QFo(LEpZu$fi>wx0iX@og2B_^twQGRQq30qIaAngx>#s2nH*HFaN3!~w#p zLpi!Wl!%y+kpXls7LT=>rr^+dQH%-nFa$e50mR`x!&%&iBPptys-*5wQ`8VpdB04u zVp5#!4s{P~e8z#|BPDQ9HB_f!L3Uqu5u`iCqQ`q*&dcqsy!_^Eq8+SH>b+t9*D;}!1Ya~qCt6s$YGRlHst zJ^9?f>|;-U%Tg*Mf)bxA4lv49o;f}@h@&9N5rJR-7_dI5pY@~1f+y`hyNt27z`$#po^uYIj1eC_KE;f833<6qd@nDh*Ip`la17H2DEKMKY1a@NDE0=O_)27jw& zeaqe;iwKJdvb+k+oIEUxgoZ`!6rU$fNou3;x#Gc%i!5qh4U^^Z&Itl_Df~d=htwdu z@C1%bD2RdHWw31m;|IPdZFjUV@A7)^9L~+&Wr+-^24e~+kT0BH@Pf20oK>X?Bs_{M&aP& z=H+#%?$7Z#$qw>)LccsIK{pUEL(wQLONC_&3kONGI5)Um4z(tcCJIA&fxiIePef-p z>p1r56A5$e%HXc0?2G5VJo1H+@H0{Cv8bhOWi(NBWc~8j%3mJ+)Ba86U+Ml_x7Gd^ z#s^i~-Q&@Y^DBdI6~IAEw4h~0^K*SZ9-Ew7y|sE{6%LFTastw_q4{?1tGUtEvyqm* zEjWkjkI@5|&bc}gqbo2pwAK+;eyL}zDLfm=KN6#lJ}fR-p%Z205sd{-sA%I#@ZX(S z4aSRGxZ;+0@oBF3bWDFbQCJc$Y~~7^U)Zok zih5(FQ$MSyUDYLQ&2d{VXY1X(5}AB9YP-gn>Q^-hlQnLt;Y=W)$4t!$6B9QzaHfWJ zddJkcpF>sF?dc@ig;CBj`k*#)#TB(Itc?ELSoGx+pdpA@`eUZS-R9#PcenEY>|0w{bAW!AY2n(2tU7}+w9yb*gU#*IZ`^jV;+IyC;O;uUrp87 zITN$0UF%GM-lDvnD{qgNk8$N=4^BkOpMeg*Sy(|0S5OniHazQ!6x75EKJr?@M_}@j zhhSqdb63L5#LW$yxnW(mV{U(ldX8?cqIvFVWcy$6dROEVXgVQOZfha*}w8LgO1&_xl8ZBMH*>Gmj< zRwoFEDDsJ7ABcs5fgCEvKIG{a)uMoNFC{tm;x#^Y;raqtWAZvMcU*EWf=$94Aqe+n zMDYD!N%u|MSqK*2K|3v#>NzmeLjS%38V%1s2bmY2KUN6wPak(7>mp+A4|KfOzPP56(^Q5v z+Zx;Y5l+*YIMK7BjB6N9!-Or{nj`ByoaT6d>?$-|7>5))gN2NfcR)$=!K?NkAX8dDQ?9Z>%+~hQQ z`Sij~>1X~UVJwof8;}-2Y8SUD=xQ7wT}tu4(>K{tVm z0BwpFs}=R`wO}0yw=H=Q09Sy$8{J z-Ch2j?rh*XpkuVk8tKAaq}Q&9t4)Z zdTX1m2y5V2rck${eWIc0f>-VmV0knKrzy6!jT4)WKRdNmwPPFLOm(X=uApvp6whoF zYkh0&YqMcnxNLoFV|jCO<0in*qW&Gz0KlyygE9agmQoRC{*1A4v=z`UoFkRAa3wAA zk|C~SXzOmY_mnhou$q z(hjb)W8=zJUbM7-H3v=*UY3BfoDJDGbsNXNY1lCz2i#S4WY0{QtPx{#MAQ5<*TC(J zglZp`-us~&wCRx&A!6$ThRqO9?0w)#EPkYEGOq?mFcD!ed>)+62GI&{K51ijS2zw( zu{qF~v_+7>eXx}u%yU$*L;^jK^c_S>1*+0V=-sz~A$$2A204dQc+OtuZrtRSKFYDn z+e^11H*Rgu-X@8KLBi@pr-lIhruY3&x|?##-0FL9A3~%gb55kVo)|dY!IrABwiN80prv?mN0VtAKRq?6u7~M*vU=E(17D_4n=xGPS06(Yx9%yVVhIb#krF$n>+3E6>MTXUH-J z4IZxm{Q{rEdf-Su;Q9YCD(ruO*0J9J&!%9%iQ&`eJvAKcA7k{lz$?N&F%ZB$1>;?p zxUNf)iK*?bXX0JYab3?rk0LKzjdi(5rvQ&4q-R&a1LImCBE$m1mLo7K0^+}p3C>^w zFi8l&4JzJ<2_@n0BlBf8_ z*4?to^x670b0%7Lc2)CcL2bfpedWT-7q;tXBJ(ePH_!uzK~S zh^g*jVe!gPqPjj_-NjXR#j8(o)u(o<&!8>q>W$s%`tY5v+EosR(pm{Ktu3UJU8R($#xL)^xpuI0MAjCf@LL?ag-VZ|tK(%M! zY$-dCGJvyCgP<@Qgb+w#cxuHm=wS<3C3=8uKLp7y#l>?RlEUJ7LKUfs9-)WlzwkgY z10LU<`f(jMG08o1g`06jCS2QNAK4yslY~!`dt4LH2zvh?pk$DhWfSl`=tQ!ZjYx>^ z{0d_1hkZHVvJqO4du0xJ7Z6xHZ}*GST~9g6G&zcxam(h)$J)Z0m^-#2jmLNwD5#ng$J_0 zPrTm=Bcc>X1jI2ghjhZl&>9L1-HOBpm!C$Q{V+~p-Xk%!q|NRV)>?DX6y*n z`8i|*Wp^%B!hG5AdBYzVIiIL)jMtvzYEQ;% z`?%V^o!WlX)G8CbL-F48T<`gv-irs!ry|FPqg6BUDmPc|Rn)0VG7w1W29VktO5$`mN0)!azF7-3gK(56s0^#ag^_}W2;K0`uF>+Cl52~I z1k|x{d80p)-yNe*Bnk?Dcl5nI6});kfW$a_?nu9$`o3P@->Ue2k+J`%;`?XI+qsrqo#zYSL`ZWluR6{^#ZDDYmbBM*VL zCn+kVp(v!qgtpruaF3ccc`=r_G^7;_<)zev+{IRez$Wy?W!UP(6g73r34SK8Cfh^I zWR_@V*^lQlbEu-c2z^CjC#-b2u+kqugm9cE>}B2p7mCEQsm#mIEr%@9!!nTZ!Sg`S zM%FgN9iLr|A!2IXDA~|Qa(j0)r=ErZ|Ej4off?xSlq3h+5oQok>IaK#c)>J1gGDL} zvSrJw1LX1a3?@Mey&r>i-pfz`iXs!3BZsl%|{%x zF2&u&I8b_JTzGgt6EFvSQNf{hL!6Yt)0YIszk#eZCs!gJrR1u7EJ>72L<62Ng z>cBih4^Nt?C=JrfDC3NP3@!|(>-wPC%j}UjGVjwqKoF#5&TK6M^dLnnu!D$UlH#hF zTyuccV}+G44!km*8%&$veG{(Gq=UILO3Q&9RYFOUT#^;)Faa?v2pt_*fOB;B-3*x* z3`(xMy?A((A?Svxbp6XL3sxtLlr%8o$?uL#f`R!wG$$knLwS9QTibemIpo292g+do z61~3y?Jy<^S42w*3^r9box_qy!`wV9_jseF;6HA0YVWx8WyfUjcf<1$FdJqxUiJ0L};^D!EiD=v9Lluf(pB z@evQ}1zEKq%i!`yKKw)w%G(KJe~)?#xk6h^(*f*Z<;JItZ<$L)*)?9)$(40R%evy` z?v;UEbMcx|fH~`P+m`mIxdRUP$q9Y%mC(zf&;91AH+(L<9KOB|$FV13bPv94{Pd-t z8!LA!>%w#K`qNzf>CNDF{Xo2ajH@4u)}Mdf*0p7Z8<5PfY?Xei&c6PeoBi7@XF>m3 zH}LX>)xMWQYjQm7zP!e+4TY=2j&((NWPM~~YGY`#DpK0JV?Ol@1yw&NgMsR>JSc8lgoQHo+g%qRaztkwpQkJc|N9^x7o0J^jqglTEyKD${Hib+Flbf?pbe zTXT4WPER0(I)mMw6PVKYitu#M_)k2+1UQuv&m9K5A(sJ-=^Bcmj#3TDCSQkjW~CP7 zl~g&{`r*IxjPP9uTQ0z3_boUNOY*Cvv>ZTO>_#=N$Ii=czpGjf&Np|1xu1sIZvt5} zR915+bJK+^HB!_)C4zC_4TSGi)i9E;JTRAp4+b%kbRoG;7gFkKY8YUUN!|sLJulN< z4CRZznE*7MKzN&;SNgz)j91-;FB5qrm-?eH27+!fhM_d&zlcAHY^xS1|4$);m(2^r zABjA>171K4oSfoaGTwcK>%J1{dOq^JD`Ngg zjCQ|esaOjN7wY4cC%DQJ(aMuMmR^u#O?j{6yqvRE7gpj8qOa0xFRjl+EIl#P$=y;W zY>rnQ=c=ajaYV+DwRkUnibs%9bd}a9M zVPFDtzyzkkL+jP+Q|m(;)e&naoOXfKTXy=9iYhtvL{H_m!?hXw3_*3cJA5RP+q|Q( zKPiT2==!@v3-C(+sevZd51RA?z1n}h=*sdqc^{5+U(F!hRjRf}6nL+K(GEPGCfzAX z=QDsdLDGSW1oz>M5=0V{+MM}BHpmf660d|yi89*ONC+8d;K(K9BqKP4O}vku5s0gT zNKKz6L{9mBWEqHH+E@bi3tkJ(Mt~hD6ZKTj0HGh6I%SS7rWUT_jGqTNNSRHe=}2As z@T+E!O%vx#nPW!ilpKdA*l={&e;XoRhd60 zYdLaq`32So^jLK@Lz257bLjk<>3i9(m`g75r!=J^Lj>lN(x{64g??%3;RpHfkwgzE zz8Bu6KRq>L7T<_{fKn^Oxb#!VH~o?9HA=?KasU+pgo7QQko~*HFfxC0+fWV zQn*&2uOuOgFtt zjZZOM*rT8W$H^fMbl@<~paa*Y&4wI?DKr4eozeosmOwVpf-C&mGg_$muTjg6;j||F z16cWZl9ii6rTAT>lsHK~g(NcZkZK6H6a$y2@qQ# z{qa{nz6D3N=U?kO|GL@w&|>|v?(@2EN7T}|s@g3mT3!BJPdM-Q&m<~p!hK)5u{yll z)Uu)aqp{7rZ(RIh``U8&@|W&>@zj=VbzryQ$a?2jLmPcxdugpRocD*_n;okI4~xs< zHQiiIcdWR3_xOoF@uY=M&eERf?!n(JoTX($$yqvgPYnEEGJ0YZBU?F3>xP-L9Di6` z5wAbV)t}rP*skx37xyJBR*3)`w{&rquFag7I|nAo1KZXUn+=@xTw)0B0(Wq5d3S2t+Oydev-Ur-Q6&{mau9zU zC-|c-d?S1%lH0nYIrh%u9E!0$resjduC*#sdusFg=5t%K52_v%M5-^utrw%#i|_3z z@cp|_j)`Ce4@TFg#Hl*{O=pj+2Qd+MK${Med-ad`bSNe>VZ~2a!mWc zoHx>>eNdN!F%S07z;NW=?J}Aa6V6<$=mXDvcLKzu8Yo&AHx676nF$`b4{OGcK zG5N;m0(8slv#u=%Jpm{-Px4LASr1&|a6wb6Ap=p033dc5sz`~4!xi{Ac<=mM z*p#m;&plC+k6r#k_|WE)i*opC(J%1RMNiJj;o|Zy@b>bPmt<(qEm(Cw$ps5pVkZ0K zG*~eG0?nA79D!5qU*OU9lU6wI*uz7QCpyrZW1QtlE2sqaP%rSL0-)tTr|>0d5qy>H ze`D*AYO=op9 zme!;pVgO?zK@5HHfd&(JsiFe@3o~vFI>E%m2g6&j7{n*fo!Qyl8tRkrCil)c=bk%r zK7Vsxt*Q1Q82geN&enJk`js;FV2i}kD?sECk5n{*c;*nJG9wJn9%B1Am1{>l*Nk{a zGm`352hXfW5$E|MzGIoOtR#jdK~Cp{v=~l_s+d$WiX1+p;P3%CEoI1PSddd;K1WDK zkh^>_B`i*;B9;XyEM-n(0cS)4;s|3@r$seER3$sE;1o$RU=N*iIAC%xm$tx?M{zX5 zZhs%46v#qokQrroW*By?!LwyL%7Urf6Y8YlP~ppl3yW$F%kVvps2CnDuyssO3H{bO zV<{cGB=C5`ru%s-TLA1g1Ih!X!ah^5{A0Vd|dJ<@1I``uL*3W(=otT9W^mRrxY=){(uqLn?x9A_QF)wUMzw55u-Ldun- zp5a#Wf9fTnvzO;;D7zCp;^jaK{~GkK>Rs z%O>8tsUphTKep?Xcdkb#;=J!)cEuHUA;NJtRQ>(Rw++|XQFxOXbR0&h5bXzO)~aSy z8I6N}gqCOQNEJeB5br-)@1@V`qt#F$|2Sp)W1qCTje0@z9jQH!eFx-GK~9RPmq9>| zWaMAtEkAMVF5^8VNn4C12u)&IQ zjz(ZeySrJxtX1&Mj*gTG-t%m$t*w%dCSC<&j?~lkgx~<$ZR0Y`9Q%YOI5r1>Vz6CX3w9l+e6a)wj;1|9=87-K{P%8Ra2w$USRx#5q;wN3hF_Fda1%FEC2DayC0gl15$%67 zZ56hDE{Tks$|MCy7C>v73X}2XAkCVOpNKx9_aJ|If91W`IyKTw1KddeO=Yd2D{t{`5Obrs?Mpc&Pt^h^2${c`QK02cXqlM> z<`eXxQ`)f;mv>&=skI(`R29>zVuhhoF9`K>YIQqv{9vTo`tFDNl=jdfXH2pNRJ3oj zhm_It`UhYjg?nJ{g7`~q-c;7+^^5vda|hIBw0Z4;e#v6*YY#!~0lXC;3IQ>%eXC#6 zz5@5?i$HO@a{s&H`TP2`b$|-s+Zq*^#!!K2rYWL<)$Up5M7)3tBK-Aprwym1jEmTC zlB|?b4Neke!+i$IN01~;V@9>c3{H?IBUPA!x#3Gnq97;CD}>X$pcpmDNt}_SOfrGt z0wh-O6y}IgUx5sXq>__WqrRexiXy1qa^YGs3^0yxfG#vm1V9P0XpKF4_E_K&csZir z*Ff34Ra8SQ{*_11{Ag9Pp58Gfm6OC>nBFgFj}r=d aiyXr+Pg#y}Ecp=Q`x$L`?tpundGK%S^}m<^ literal 0 HcmV?d00001 diff --git a/app/keyword_extraction/__pycache__/lemmatization.cpython-314.pyc b/app/keyword_extraction/__pycache__/lemmatization.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6875150babf0822ad36d6b1f5d15407a7d50e73 GIT binary patch literal 3455 zcmcgu-*Z#P72f+pLP!SLjvSK|%*8ffwP2BinUIz+HO8JHCAM+_r4JO1B3)`Z^1Yh9 zQd09I8`9#!;I`9Fe}rlBz(d^W1EFzkV?rnY!QMJ8&YeCm)2F^ss82cwRYDKLot2rnYX0j4H%}nK}Qc`}P zB=wrvdcA|~uQp1WskSpyYiEV#l5XnI15yu|LFgN#9yCMHH%fhjsdTY;`0UZ}%MQ=E zM!^~z%arnG@j13pF69gPQr>cnGY&V*vg_tEc5nDt)d>Bs%UOAQeQ+qNU^Dblq1|fu zR`)?vSc;ui51eN#hrX@sC1pf0l~Wj5E2g@xC)6%Bqy3Big|E{pevn(GGPh?GUvo=5 zuJ}Q03Y&Fltxi9c7;(m|gbiV59ox0|YZ*IN9!rdkC%H3fWlL@%=VaZzu9e~0kwm6g zOuTB1lk2qgqY}?#>Bx!C{ea%$xZjjcXKdRkWlB~$%_CTABR=;@hzk3ZZB$}+NnGh) zq3UYCY^`FH$sa)qmFkbsSI?kPVc%fuM@`ZXYA7kTzxoSLvpuv{wG@@qLm|+`47Qz} zU}be1>tf&0wz0etSITpk78Y-L6)`Wai(BG7G4D+pUd21-O?VUH2D)YqD2q@qZipL( zcwb!eej=`kMeh>yTVlcR&Wl@caZ}9phQ+U>`MiNSv*@}B>q&T@L;YiTpC$+Jc@AsA z2YM&GOM48t`kc59H`lx=jLc#mFrV<=klUh#?%zpoIIj7jF^3mN(!A^kiW%-&e#E95 zFQn<><0=o}bbe3{+HSjeO>Uq-=8cH`yQbecD%t@VLea z7&ycT&j7MPH24OHxWeUbCAQ_EB=n?+VZ4sNu(InegZN|D3RUlQ$rf*d&FAn(o`Exc z^M1(AEGf0N;*k2iX1c--R@92BsU(St4JD=3th?ZAJHDctXK}T>48yQ^ zvjMU5M1(aYUI!8{1O9;FqGZF3DxNPJJ>Gdjt(f!90kL9E0=!Bd;>Zoa_>DS=;-JVH z-#{0y@@=D2{0US#C*Fl?m;&WQ5U~KCx1hc6U0ee`2qES>jlG8+n6dES3~+LSHtt>O z4Eu@^5Bgfcvi$%a#(dEa6r9&Bj#d$`w?w|Lk2-lfu5-eUACRy18hx7URLP}fw@ZNB zWf)Z|ZL5#hsFcn+g@Tin5rTKXn0&ff2)u`3*3|mTq1#P{*JLa;byfyGI@q_=et2Qv+K~^p#NX-u`t&_P-p@QYj`w#7*hM)3oq zi{52&N^4XJTI-udz`1xIIeWvtK6LU$^TgmG89Vg;;h`6N{pipM$f2Y;?CVLh{}{=E zFrhrX5^J`N7iRNf+ci;nBJo+;ZhcSxg!YzetX1BaUPZn|znB;&o9 zZQyf&2&ZGJTNnM+%ne7-NhK2S800ktpuYQQ|q<8%Y}p^@)Fhu?zSw`AF+{|Uw@z*w`9SbP@K5I=|^%2wQ`AXYR*QSPdmqOXRS(sCyf jyB+ELYov22(p?GN(~*(idmL%ey)Ag0KkWeorOW>Urw;9W literal 0 HcmV?d00001 diff --git a/app/keyword_extraction/__pycache__/paragraph_processor.cpython-314.pyc b/app/keyword_extraction/__pycache__/paragraph_processor.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fcb67bfcf5bf43e3d647d4e812d8e30d6d5eb93 GIT binary patch literal 4646 zcmcIoTTC3+89uW!du6Y%zybyagCT3fF4)K+#C3{OBW%|WSv!j>W2WQXF^sc2yE!u( z8`K9>Rq7^{S`sBS=|x$qiDa5Q;3vy{OQlxwLse5CY(|z!)k;;WK7g@A`quxPnO&}p zU(zG(f9_}g%Xj|&KYPUMaUm$D5^XpB=t1c3q*E%K+1NM^jcF7^8X7|}<_x1TP+QJe zhFFaq;xw)qRihRZV{1{2>$MoWG;54$MPcjIplevf;y4zw6MPnD5|XOQ$~d3XJDnw^aw zKtlvpuOf{Bb}eHLjjcf$U|Qp1?l?Ne#4NQalKuqjq!D zZjL(WsM8#E#9W#)jx<-S0>t&|m1Sv%4Pi5Qs%uD8k|_{%oBWyUe72;QouVcfkb?Y3 zAvPf5ZL|Y9!Azq#8ng8M2%#j59zd@$6HJV`$Xo$?5E``r6YQ772D&#b-V&(#1XZ4r zbbIDa38%ztn9*5KE$Gt*s!xlW?h%AUN>tS(C{_?uvisn)J{q0Kq@}0=L`h^6Rl;wA z7`b#bJqe=SkP@02O=c2mmnw-kF%cEB+34%iz3)1s03yW{C~M0$C7S`)F4q7aSNmN&l%2Zm=g*6` zE1Ptr3;T+uq)e;qoHgAqX?Naw?HV#e?gIq_XMJ%U0hy~>Bh>?zR-b7PU;e`+oiN1O zjteX7rMUPlG{CVjS#}g!Mj0kd>?$p2q^IQAvRs_=qt~oCO9bUCgNG4%1NE{{gkA{0xB}IelDXEbu z&y`5aOIMFv8C6G>t49DT7A)yj>RF-Eu$&Fsa20TaYY7Pf(fy_Uf}|wD{i?dNG@|N^ zs@h-?pbG$R04zw41CVJMVQXQ>K49B#AvN5$;K*sT+HmBNBlI{}b9?CBp+}8fi^1N! z^NZHbMPL7&7iPu#_IvjG&i9-%%x(KS_MbcNys+Twf8^-@&tH7~Dv4OXt{w1m4}w6x z5x`n@19r{?th4OCsKRg^e0z??1l|;_nBeN!kUapa)G!bV**z`o9}&#QttCE^ zzasq@GfbS>t`fJF&##Dc!^8*dDltpk9%JJbF)nT&V>$^8%o4N49Z{~dzZuba;t|b~ zwS&xVkl8a@9^LvL$a@x{tT*o2Cat%0+Ac|n7ur?&`ax0<5hQgF2X;+DcMQe7C942$ z`+ud(7jWV^2mCan^A_R-F1fSTOP;I^yv9B{*|JJA8o2>l+4lL)?PuD~{oC*=+pO{8 z|H~TN^jJpK4*PHr5^PgHbQ$hU+!k(W(r3oPEJ+AU37{)RLxmZCNhU~QC`nmV{-&%= z@Eu03BPTmL`3@3iNUdrj*2=DmjDVxt>brD?*Bzye)G_MboqKfm_8nm-J_st-osyDE zOBk{jolQv!4ahooBO@!i4H6;PqC2nz370A*Y3SB%GD)Gdu+Q+pBz_q#g(v6P3^8?2 zB8M^fbR%)oy`?UEV_ciShsjz-)*U71tNThV!C0cZ3~6Yf4Va273#{ti(vW##3!JEu zsA{-*lH1T?`1=OKjhCIE;rUHoBmdLw@4KG*KmmLVq#a&VUAGeKS_pRCl^27j^3H$y zLdB};{F&82Ro+$%)#p9My867gSQX4W{?X7@_}=FY(YMa!2Oc-I6rTHKI{(V5uW{KI zoo`z?dV1mL=_TLkKX(-y`42CDaJi6PZ0wzJ75z20J@0sm!KQ+v7_6T?TMV`pCguYl zr;5RYg@H9l94nt9YoKb)jRJME1Mhog9BVdosN<2prPy+C#{0OgrO@?xUH8mDvG&lX z?A?J+Y|FKg`L_F=_d1tqPZmSrl~DIWsCzyzKd=-!UJSK=+Im;~qRER8ynSK)d32zA^8|9&73&Us z82TVo=vu7n&Yvy1D}Q$Wt@A~H{p=73Gz(Jkh3D?R`f+E`-}Dh%IP)7X2-4sMK|DVA zmcJ(N`R3BNJy8)K0-#{x&7U3e;sjXbEUl7+JRoz&)=2fBl~xaf50ihYrxSbQ9f>J4 z-u(_5(`Xc^QTET!V(_+$GreX!`??G(%Z%DfXTc6L>OmAEdG@YCeLF-Nh+6Cz1CfV` zS!1>+8(~^W1m4FZ?~K6U=j>B2x!}P>9=`k@PiI^u9!p|!lD>q^7Zktsf{M7N*irQe zCg}rp0EU|i$f|HXmr4oc%XSI!J95b&m(E?oJl-*XH`RSpQb z{l=9Kn9sDWbM>};F8ZET)ZPJzh9=*SQ!xx)SVL2=*-o`}5AH zHsto@Cx3Es_R>mY_d;X$QbqSF;Cq1J+gbFt7A~#XxoR(@@j)LDitEanl|auzpl3e3 z7#PgkR(;hb&T%Dt5&j#HpMn{`JPI9n-dZFjTtOZ~ikK#^OP3&|Gs#?vwBg?8VA?JS@TL_69E78&tH(rh_!JPm8Xm{+V3iYss6jq`EvHE;><>w~f~e$)hkK#2V^xAYc6$RP_b-{3Zvk=m~kqZq~p9<_USlY`P&Y zd_t1LO^t#3vPmuty)yKk(1XC*kR~9*Db*pi5X& h2}agfhGEt%7KYn!Ajb7q)VRrgpJ8e?5TSJce*uGa8!-R? literal 0 HcmV?d00001 diff --git a/app/keyword_extraction/__pycache__/stopwords.cpython-314.pyc b/app/keyword_extraction/__pycache__/stopwords.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6ea52b35e9e7e78b2685006efc930940216e52c GIT binary patch literal 14746 zcmcIpd05rgng0DQ7X(oe5fPD1L_kD9L_kF2u5rgLRt>#gE^s5y57#xnRb@W)|;lC%(R{5{ob=&w0Syz%ridE`~BYY zopZi(&i8$1;aWdq`cw!1*7+CSmnd_bpGd*{%hcr6U*&d?*Z^z4LqG@! z1H-`mKm>>aF(3|{07ifZfKlKi@Fw8Rz=ObBfQNunz-izNa29wg@G$T;;1S?a;2iLF z;4$DGz~jIZz|Q)N#gZ-{&*Mk?*`rjycc*M@P6Pa-~+%1fe!&s10M!H0(=zs z81Ql66Tl~dXMicdG$0e03SM}V~ZbKoz4zXbjY z_%ZO;z~2Br0e%YnE%0~1-vj>u{0#U<;Gckh2L1*3SK#MB3U~$h1@J2HOW;?)zX7iS z{|@{I@N3{Vz;6Mu_n*Lj0snp6-RO2ZlrE)*GK11fiTI%nq6dkXLL!QgIh2SjcrGQP z4L+T624x=QOv-%9S(F8ovndNHC)=DuO%bKCIhVYcavo&~CGrO&mQo^*z{@EsD3Mjr zETF8STu50>xrnlcvX-)raxwgZc%WxJ$0KkNV0Q&*(Lzccs`6ZOU z1AGm53HTB4J>W9%0&oSm3XB2QfER)5?0Es~`(zg>S5RJ}e4Y|Eu&-bN61E^=2@dCHB6CjI37!8V7*uXj2uJB6UCB3QW_1!WI}#lJSwY(eJnqe^ach|_Kpb8 zgp9!c$Ux9Pz|SD6VgvA%$XGHji^jW?T!W2km2jFwEDbesy z0IHK&p>Ty%;d?CHJ1RcK^h4ofW=I+*z4yftATKz46%Gwz)?x!W7RDgHK?#Fza4_i= zgA$|BWF|*JA0M2Z(H9uOVnpv9ibLTY3QNQX0}}YG!C)+gSY!-hR4+sniHRus;4dE{ z<`LJ7APU|Q3843wBwgmQfaIEYz;{A?i&_ZeAj+cm^kQPY;Q-XV!QNzMZ&V!Cw8Ok7apDoB=8-AarnE-Q*kQ2F~6+A9;M5?9j2 z@rl!k7ZX=lJMoCFy&9i}#;CcTxCAp-C*GlY-$2h75|<~=s^+Jasl=G79Y^zV7#mkb z_N47>8Tm$H46t>Vr77&nK#5y0G(K?#EiT2SF^ZTm>~bZS(eS*Y7cl`fIZO<*nK+GJ z+67~%!QbBwtXyqKh#zuoNjYPdlrLgR7ZT5lcM=ySPQmg_8Syj}&r229O_-b^ZDk2E zw<}oLSe)I(m&S=$?!>A1Y_?Qxgl}XK03q1J2Vgfsa|97+ZY2Ugw~`1L&Ml3PHe(%FJP6_Uc$;Qn-mZq zh;QLjRf_=9fTZojBSI4wCmz%JLJwitSdyd&YSB|o<~k{p{o=$~3?K=j(;2ymaL58^ z6*WkCDbfhCAD&}E-UP)w?Ybr&mJyLFY7@cD2(y{GcwM&2g~T=Mgo#IWEhjGVlZEL~ zWtGMYte1k_2xC~Sgm98V7Oz7rZkaSunUxfg&$+{{zqF9>V-YGia3BR7d%SX?F>P`%x-&xaB~&35(pclBpL_U?Vjp4sZplqYmaLPAhtel+|Ms z4=ddb*j1^43B#1>rWT`3U9+a(5n6>QP-V*KaM0*rz?AIbbhn8|Z8uO({IqVu!sFtt zi8E}Z>7=lUrQ3>^cv5g9@<3ype;J3^SsrLQ{l+ACs1@&EIPngrOiDBK+jUu;YQ*iOD3`A@1ld5h zN?0@*Q$%(m&ytJk^MX>#F~EjMc(qknUGdQLkyxOSDkZmvM_+t?^#Sk(b0*2HWX)w6PKEUmo`!lNa%##l^<9P24PG|)5f5c`Fwo_4Qz5a% zk7`F_z1;80k1&I?2}|V7aULV$f)a&hfw(j|EPc(UlB9T=62{_R+<$ z+lyD!g{8uS!*X(w_Y?OgpBSjcX6L%-Qr7S~Ek%8MON{<{Eve>N^gIG?-6M%1MP~hVc{iB!V}Nc~Nbo1MkaZaa(@*^Q zI?A!zlseRMG0Rw-yb04Cs`Q<&&C`;-_a~2OmD)#RcicBxEw9|k*AO>|tdv*uZB;xJ z%oD>F?8{5NRj4wfZU&vsXZXyOvqSuaW8yI=bntJfqnsOU;!)J9mmU{&TIPlGji-*>QMW0O zKe)ko5txHYdQVcQKOAt&VKXTkYD@4q;B;hWm=M-^jcKmt2eVINKVhEcj8L3txJexmXQik@pK!uLpnWM|}QRFdS;MKlm*j9!*Z`>G6f|vy3km z=;?`E!#HsuS^#m%dCl>-%U==k^m=|?h1%h2g~&hok7caP`|1XeQ%-tiyTu20{5vqI zzS~_gFI~M{Jb-^yPMzmu-R3~-j-Ms*<3ol1#WK)aVH~wIBHijd$(+6WcI`g6t9!%V zWJZ4=_M98Jj0(DKQDh7>IUVXAyYk2_=(x}#kWNQgfP|1ZAJ-MxS3`hEM??%TR+XXGZP7#W9{?E50|{Uqx>+Ir+A zH00vtCk2VZobzkWt$C{bnZp-mf46$Y%eid{Z(G^}W{1gHDcSYvDt7z*JFMH2&dKhK zBRbC|Sd9d>+l@dDgCE7|{mv)il?W%F;jCP(akI^ZtX?K)70Rk$b1)(-^y z$18$;6}}2tOGPjWX3~8rc1Y%67{78d@3FS8qodj)GM`0ykK_iUc8f&5i3%=G#pcJk zdFNN2Tlv_kr@Nk=`*O*$iQHug?=r4wC;Y>Gn3bH;(}M#D2gwZ?;Hx5Jr!mm9_?*Ko zYENdb8VdKu2Lr1k-$o-@Q}kQ7=2IE2>%QXgxZc!MCo?DI)aPdXI6F6$Dc|Y&>RT{Z zedm>_Z^c6OU9?zzmo%yGvNrYY=u+P`8`XF7HhxoCPR^{<6j%nEJ>RghYQt(84QuW& ztaF2F=q7ik=GkH>M+{|)AcW64J)ZIY{4SKYL^(c zbh%>lot){ZQn51M$(wB${HJyBpJr8cij_%^a_Ld#)W zR@bE1A}1$5RUcnVlUHC^;XK3UR~WXS z#<03Z!ws}>hQ>9&TQ(&5N|}4747AdjIoB{8rdo%&YgXG}SmO%A+Pe%}gAJt{ zZjqyqe3g?kGu0)=S2?qa4J)ZK45_Vam$Vqx+G$u9PFG#MNp?H=I#(vV-W8+ZGYiwa zs4UGfZQG`HX__~+rFqBdH0Oa}du$Q+sL~dWx0U08V?4_-p5++Na*StrQ)`;Htxog1 z~>dqb7*)fRslA*BI1U}l z=T)Y8)si&F@oU?xSe53hH@fMQGCJU9wdQ(zISsLHVFX2V+6 z7`E2khPQ=kxx+nV`C&I``EfU{Io==nsT^sYEhnvJrS*p4h|{`O91fbUlZ@AVj~lSO z-yPK)FOcF?u9%(bl;Dk_SToXC)2)0>DD^?tZ}kB&!8X&JoT}7xX)(>I?lNo*7N?b5 zoaS7d=EvRpZJVe&qWKKT+dMHhL$X%0!V<$us|~~3P1oYs)vOaUC&l|=da6*&%oa16 z!Hj05Rfb`kXU-Sd zmJf;N*#?iMqEv|(p63+T7*^M6SQ}=nt2c`yG(RYbrFp;PD>>dU(^HjVtU_Kw!d|Ix zib@SDuQ9By*|3%l!^Ho(%X--+n(uHAT7F0zrS$=KNOScNlELs0Do#}?hPz3#>UzT( zS`BN%V7i)vX}&|EqWOMtp5{GnzvV$UtT|p*`Kf9dZlS!BG{Y)2!!@oMl2x-#yl^z# zD67<*tJIvU)SRm%$D^z;RU?Bfl0!zb>Q=+runV+utIS;U9ywVwKQ8;19B;UyRGk>B zb&9JEt7$b%+@q`M9?iL{H9st|(44a*Uo0-EhvC#>r=XbZmBmg;rDAx!<)xZs*d}?h zYF1os7;Yoc;cbU!qONVkJ5Ke$dqT4=I8is;$hVv7v{$yL*864MY>qcwVX8$;EOUy= z4a2Kf>kuW)&0e!1-bR_VChDJ(avvca&%PO{W;rweaR z-EfG`wsh2un$jj1 z$WLvi0q}zIG)Kg&zOgOMJ7CyW@Z7eg{gQFEU1F9r%;6l7g#RK3)HHxRe>*O5^y>N-CZM0cdXREjyZCy}O&z2r^cX@P49=XMM zjmkjaaGmANFm4OXL|Zy4TSS%Q(VCCPsTcsq1!8%5o#H9z@uP>;w{~g$0rbsF9b{kd z{E{>;tx0oSv$hRhA&TQ6jByXKFF3}v9OGJ!aV^KViobz<53?^g`dW^@mZPuYZ$$fo zl#lJfaXMO#pj(a;$MO#3oh|V4v85gI041*S?{2Ua#s4~s*D6@Kb zGugs=xkgl}7u}0e0d@z+tH^R>kmVh?7HokRgDUl*dughl-NDOS)4UD&%!Mugdou86;km^>@Mcd;txRQR - - Document Parser - Extract Text from Documents + Enterprise Docs Search + -
-
-

📄 Document Text Extractor

-

Upload documents, images, and PDFs to extract text content

- -
-
📁
-
Click or drag & drop to upload
-
Supports: PDF, DOCX, PPTX, PNG, JPG, TYP, and more
- -
- -
- - -
- - -
-
+
+ + +
-
-

🔍 Extract Text from File

-
-
- - +
+ +
+

Login

+
+
+ +
- - + +
+
Don't have an account? Register
- +
+ \ No newline at end of file From 4d0d5687ffe574ccd3c06fc584e76d7ae76e8990 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sat, 13 Jun 2026 00:37:21 +0400 Subject: [PATCH 18/34] refactor front-end --- .gitignore | 4 +- front-end/app.js | 288 +++++++++++++++++++++ front-end/index.html | 591 +------------------------------------------ front-end/style.css | 303 ++++++++++++++++++++++ 4 files changed, 596 insertions(+), 590 deletions(-) create mode 100644 front-end/app.js create mode 100644 front-end/style.css diff --git a/.gitignore b/.gitignore index c14bf69..274ff08 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ target/ share/ .env* pyvenv.cfg -tree.txt \ No newline at end of file +tree.txt +__pycache__ +uploads/* \ No newline at end of file diff --git a/front-end/app.js b/front-end/app.js new file mode 100644 index 0000000..6eb47bf --- /dev/null +++ b/front-end/app.js @@ -0,0 +1,288 @@ +const API_URL = window.location.origin.startsWith('file') ? 'http://localhost:8000' : window.location.origin; + +// State +let state = { + isAuthenticated: false, + isLoginMode: true, + user: null, + documents: [] +}; + +// DOM Elements +const views = { + auth: document.getElementById('auth-view'), + dashboard: document.getElementById('dashboard-view') +}; +const navActions = document.getElementById('nav-actions'); + +// --- Utility: Toasts --- +function showToast(message, type = 'success') { + const container = document.getElementById('toast-container'); + const toast = document.createElement('div'); + toast.className = `toast ${type}`; + toast.innerText = message; + container.appendChild(toast); + setTimeout(() => { + toast.style.opacity = '0'; + setTimeout(() => toast.remove(), 300); + }, 3000); +} + +// --- Utility: API Fetch with Interceptor --- +async function apiFetch(endpoint, options = {}) { + const token = localStorage.getItem('access_token'); + const headers = { ...options.headers }; + + if (token) { + headers['Authorization'] = `Bearer ${token}`; + } + + // Do not set Content-Type if body is FormData + if (!(options.body instanceof FormData) && !headers['Content-Type']) { + headers['Content-Type'] = 'application/json'; + } + + try { + let response = await fetch(`${API_URL}${endpoint}`, { ...options, headers }); + + // Handle token expiration / 401 + if (response.status === 401 && token) { + const refreshed = await refreshToken(); + if (refreshed) { + headers['Authorization'] = `Bearer ${localStorage.getItem('access_token')}`; + response = await fetch(`${API_URL}${endpoint}`, { ...options, headers }); + } else { + logout(); + throw new Error("Session expired. Please login again."); + } + } + + if (!response.ok) { + const err = await response.json().catch(() => ({ detail: response.statusText })); + throw new Error(err.detail || "API Error"); + } + + return await response.json(); + } catch (error) { + showToast(error.message, 'error'); + throw error; + } +} + +// --- Auth Functions --- +async function refreshToken() { + const refresh = localStorage.getItem('refresh_token'); + if (!refresh) return false; + try { + const res = await fetch(`${API_URL}/auth/refresh`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ refresh_token: refresh }) + }); + if (res.ok) { + const data = await res.json(); + localStorage.setItem('access_token', data.access_token); + localStorage.setItem('refresh_token', data.refresh_token); + return true; + } + } catch(e) {} + return false; +} + +function logout() { + localStorage.removeItem('access_token'); + localStorage.removeItem('refresh_token'); + state.isAuthenticated = false; + state.user = null; + render(); +} + +async function checkAuth() { + if (localStorage.getItem('access_token')) { + try { + state.user = await apiFetch('/auth/me'); + state.isAuthenticated = true; + } catch(e) { + logout(); + } + } + render(); +} + +// Make globally accessible for onclick attributes +window.logout = logout; + +// --- Document Functions --- +async function fetchDocuments() { + try { + state.documents = await apiFetch('/documents'); + renderDocuments(); + } catch(e) { console.error(e); } +} + +async function uploadFile(file) { + const formData = new FormData(); + formData.append('file', file); + + showToast('Uploading...', 'success'); + try { + await apiFetch('/documents/upload', { + method: 'POST', + body: formData + }); + showToast('File uploaded successfully!'); + fetchDocuments(); + } catch(e) { + console.error(e); + } +} + +async function deleteDocument(id) { + if(!confirm("Are you sure you want to delete this document?")) return; + try { + await apiFetch(`/documents/${id}`, { method: 'DELETE' }); + showToast('Document deleted'); + fetchDocuments(); + } catch(e) { console.error(e); } +} + +// Make globally accessible for onclick attributes +window.deleteDocument = deleteDocument; + +// --- Event Listeners --- +document.getElementById('auth-toggle').addEventListener('click', () => { + state.isLoginMode = !state.isLoginMode; + document.getElementById('auth-title').innerText = state.isLoginMode ? 'Login' : 'Register'; + document.getElementById('auth-submit').innerText = state.isLoginMode ? 'Login' : 'Create Account'; + document.getElementById('auth-toggle').innerText = state.isLoginMode ? "Don't have an account? Register" : "Already have an account? Login"; +}); + +document.getElementById('auth-form').addEventListener('submit', async (e) => { + e.preventDefault(); + const btn = document.getElementById('auth-submit'); + btn.innerHTML = ''; + btn.disabled = true; + + const username = document.getElementById('username').value; + const password = document.getElementById('password').value; + + try { + if (state.isLoginMode) { + // Login uses OAuth2 Form + const fd = new FormData(); + fd.append('username', username); + fd.append('password', password); + const data = await fetch(`${API_URL}/auth/login`, { method: 'POST', body: fd }).then(async r => { + if(!r.ok) throw new Error((await r.json()).detail); + return r.json(); + }); + localStorage.setItem('access_token', data.access_token); + localStorage.setItem('refresh_token', data.refresh_token); + } else { + // Register + await fetch(`${API_URL}/auth/register`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username, password }) + }).then(async r => { + if(!r.ok) throw new Error((await r.json()).detail); + }); + showToast("Registered successfully. Please login."); + state.isLoginMode = true; + render(); + btn.innerHTML = 'Login'; + btn.disabled = false; + return; + } + + await checkAuth(); + showToast(`Welcome, ${state.user.username}!`); + } catch(e) { + showToast(e.message, 'error'); + } finally { + btn.innerHTML = state.isLoginMode ? 'Login' : 'Create Account'; + btn.disabled = false; + } +}); + +// Drag & Drop +const dropArea = document.getElementById('uploadArea'); +const fileInput = document.getElementById('fileInput'); + +['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => { + dropArea.addEventListener(eventName, preventDefaults, false); +}); + +function preventDefaults(e) { e.preventDefault(); e.stopPropagation(); } + +['dragenter', 'dragover'].forEach(eventName => { + dropArea.addEventListener(eventName, () => dropArea.classList.add('drag-over'), false); +}); + +['dragleave', 'drop'].forEach(eventName => { + dropArea.addEventListener(eventName, () => dropArea.classList.remove('drag-over'), false); +}); + +dropArea.addEventListener('drop', (e) => { + const dt = e.dataTransfer; + const files = dt.files; + if(files.length) uploadFile(files[0]); +}); + +fileInput.addEventListener('change', (e) => { + if(e.target.files.length) uploadFile(e.target.files[0]); +}); + +// --- Render --- +function formatBytes(bytes, decimals = 2) { + if (!+bytes) return '0 Bytes'; + const k = 1024, dm = decimals < 0 ? 0 : decimals, sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'], i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`; +} + +function renderDocuments() { + const grid = document.getElementById('docs-grid'); + if(state.documents.length === 0) { + grid.innerHTML = '

No documents found.

'; + return; + } + + grid.innerHTML = state.documents.map(doc => ` +
+
+
${doc.title || 'Untitled'}
+ ${doc.extension ? doc.extension.toUpperCase() : 'UNKNOWN'} +
+
+ Size: ${formatBytes(doc.size_bytes)} + Date: ${new Date(doc.upload_date).toLocaleDateString()} +
+
+ ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) ? + `` : ''} +
+
+ `).join(''); +} + +function render() { + if (state.isAuthenticated) { + views.auth.classList.remove('active'); + views.dashboard.classList.add('active'); + navActions.innerHTML = ` + + ${state.user.role.toUpperCase()} + + ${state.user.username} + + `; + fetchDocuments(); + } else { + views.dashboard.classList.remove('active'); + views.auth.classList.add('active'); + navActions.innerHTML = ''; + } +} + +// Init +checkAuth(); diff --git a/front-end/index.html b/front-end/index.html index da86a93..cb02dec 100644 --- a/front-end/index.html +++ b/front-end/index.html @@ -5,311 +5,7 @@ Enterprise Docs Search - + @@ -362,289 +58,6 @@

Drag & Drop files here

- + \ No newline at end of file diff --git a/front-end/style.css b/front-end/style.css new file mode 100644 index 0000000..1605244 --- /dev/null +++ b/front-end/style.css @@ -0,0 +1,303 @@ +:root { + --primary: #6366f1; + --primary-hover: #4f46e5; + --bg-gradient: linear-gradient(135deg, #0f172a 0%, #1e1b4b 100%); + --glass-bg: rgba(255, 255, 255, 0.05); + --glass-border: rgba(255, 255, 255, 0.1); + --text-main: #f8fafc; + --text-muted: #94a3b8; + --danger: #ef4444; + --success: #10b981; +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; + font-family: 'Inter', sans-serif; +} + +body { + background: var(--bg-gradient); + color: var(--text-main); + min-height: 100vh; + display: flex; + flex-direction: column; + overflow-x: hidden; +} + +/* Glassmorphism utility */ +.glass { + background: var(--glass-bg); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + border: 1px solid var(--glass-border); + border-radius: 16px; +} + +/* Navbar */ +header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem 2rem; + border-bottom: 1px solid var(--glass-border); + background: rgba(15, 23, 42, 0.5); + backdrop-filter: blur(10px); + position: sticky; + top: 0; + z-index: 100; +} + +.logo { + font-size: 1.25rem; + font-weight: 700; + letter-spacing: -0.025em; + background: linear-gradient(to right, #818cf8, #c084fc); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; +} + +.nav-actions { + display: flex; + gap: 1rem; + align-items: center; +} + +.btn { + padding: 0.5rem 1rem; + border-radius: 8px; + font-weight: 500; + cursor: pointer; + transition: all 0.2s ease; + border: none; + outline: none; +} + +.btn-primary { + background: var(--primary); + color: white; +} + +.btn-primary:hover { + background: var(--primary-hover); + transform: translateY(-1px); +} + +.btn-outline { + background: transparent; + color: var(--text-main); + border: 1px solid var(--glass-border); +} + +.btn-outline:hover { + background: rgba(255, 255, 255, 0.1); +} + +.btn-danger { + background: rgba(239, 68, 68, 0.2); + color: #fca5a5; + border: 1px solid rgba(239, 68, 68, 0.3); +} + +.btn-danger:hover { + background: rgba(239, 68, 68, 0.3); +} + +/* Container */ +.container { + max-width: 1200px; + margin: 2rem auto; + padding: 0 1rem; + width: 100%; + flex: 1; +} + +/* Views */ +.view { + display: none; + animation: fadeIn 0.4s ease forwards; +} + +.view.active { + display: block; +} + +@keyframes fadeIn { + from { opacity: 0; transform: translateY(10px); } + to { opacity: 1; transform: translateY(0); } +} + +/* Auth Form */ +.auth-container { + max-width: 400px; + margin: 4rem auto; + padding: 2rem; + text-align: center; +} + +.auth-container h2 { + margin-bottom: 1.5rem; + font-weight: 600; +} + +.input-group { + margin-bottom: 1rem; + text-align: left; +} + +.input-group label { + display: block; + margin-bottom: 0.5rem; + font-size: 0.875rem; + color: var(--text-muted); +} + +.input-control { + width: 100%; + padding: 0.75rem 1rem; + border-radius: 8px; + border: 1px solid var(--glass-border); + background: rgba(0, 0, 0, 0.2); + color: var(--text-main); + font-size: 1rem; + transition: border-color 0.2s; +} + +.input-control:focus { + outline: none; + border-color: var(--primary); +} + +.auth-toggle { + margin-top: 1.5rem; + font-size: 0.875rem; + color: var(--text-muted); + cursor: pointer; + transition: color 0.2s; +} + +.auth-toggle:hover { + color: var(--primary); +} + +/* Dashboard */ +.dashboard-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +/* Upload Area */ +.upload-area { + border: 2px dashed var(--glass-border); + border-radius: 12px; + padding: 3rem 2rem; + text-align: center; + cursor: pointer; + transition: all 0.3s ease; + background: rgba(0, 0, 0, 0.1); + margin-bottom: 2rem; +} + +.upload-area:hover, .upload-area.drag-over { + border-color: var(--primary); + background: rgba(99, 102, 241, 0.05); +} + +.upload-icon { + font-size: 3rem; + margin-bottom: 1rem; +} + +/* Documents Grid */ +.docs-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + gap: 1.5rem; +} + +.doc-card { + padding: 1.5rem; + display: flex; + flex-direction: column; + gap: 1rem; + transition: transform 0.2s ease; +} + +.doc-card:hover { + transform: translateY(-4px); +} + +.doc-header { + display: flex; + justify-content: space-between; + align-items: flex-start; +} + +.doc-title { + font-weight: 600; + font-size: 1.1rem; + word-break: break-all; +} + +.doc-meta { + font-size: 0.8rem; + color: var(--text-muted); + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.badge { + display: inline-block; + padding: 0.25rem 0.5rem; + border-radius: 999px; + font-size: 0.75rem; + font-weight: 500; + background: rgba(255, 255, 255, 0.1); +} + +/* Toasts */ +#toast-container { + position: fixed; + bottom: 2rem; + right: 2rem; + display: flex; + flex-direction: column; + gap: 0.5rem; + z-index: 1000; +} + +.toast { + padding: 1rem 1.5rem; + border-radius: 8px; + font-weight: 500; + animation: slideIn 0.3s ease forwards; + box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.5); + display: flex; + align-items: center; + gap: 0.5rem; +} + +.toast.success { background: var(--success); color: #fff; } +.toast.error { background: var(--danger); color: #fff; } + +@keyframes slideIn { + from { transform: translateX(100%); opacity: 0; } + to { transform: translateX(0); opacity: 1; } +} + +.spinner { + width: 20px; + height: 20px; + border: 2px solid rgba(255,255,255,0.3); + border-radius: 50%; + border-top-color: white; + animation: spin 0.8s ease infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} From 94e165db5537c9c035b4175486c1746adc461aec Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sat, 13 Jun 2026 00:48:20 +0400 Subject: [PATCH 19/34] Finish phase 3?; Update front-end --- app/__pycache__/documents.cpython-314.pyc | Bin 12282 -> 13481 bytes app/__pycache__/schemas.cpython-314.pyc | Bin 6228 -> 7368 bytes app/documents.py | 28 ++- app/schemas.py | 19 ++ app/search.py | 144 ++++++++++++- front-end/app.js | 234 +++++++++++++++++----- front-end/index.html | 27 ++- front-end/style.css | 155 ++++++++++++++ 8 files changed, 553 insertions(+), 54 deletions(-) diff --git a/app/__pycache__/documents.cpython-314.pyc b/app/__pycache__/documents.cpython-314.pyc index d6976aff074a662dd38bb678ac5900bf09f4692a..eeb3b4961a489ab619cd42139062b86febc3d93b 100644 GIT binary patch delta 3128 zcmb7GZ){W76~EVi6FYX|#E#=Qj-T_#$s>+Y2T`(tThh{!NGvi&rD|8txbY(~v14=J za|e8A3`_#l3dHU{ter%+57XeoG>V6sLQ2(sZ};4GDtm(um8NY-+Z5JL(I!pWInRmH zd0}Nw%JI4Pobx;X?zuPrOkK@7N1P5Dfv=|!Ir?R0-8zGZq~YET&rk|Yh0gd^dM(nRaH0ig*|!ihFz zDA^j1*VbR9A(9+QxDxFfx_}Ul(}VodP)A2c(wA^=2$E(1_V6MdLz?5DlxDN@3F%l#k`=9OV=0$% z%CsWo`8m_mE|F@E`4ca(Y$he^>Xi>o)LSmI`SpAWR8ECaB?2Ko7x;fXM? zTJw=?PD&Ni$~+r_wP?`nnZj%-FD)vmY=-Sd9b1tVW+hq9F3yPt7UTC@&eCoCm?J_b z_+^KGtP3&Q(YllMA>WUr<_S4;q@%N8C#n$nEYafN*g_jQN#h-RHF zEGiHL+r=Z!aeo9ATT8`UA)QHW1jr7e$p7lVX@1WK zX;fU@e=_W4sO;K0QUrS$36`2|5i)B-L5#-&{^5CKvp_^kLmrw9OTXT_6fXw?G{P?j zeCAg``&4BmaGMV5X9?5StgJSAn{|M~>;Vbe%ij!ox*vM5-ryevV~|#h@RZRbqk)2? z-Xm7xe-H=wxqk1`mVH6xt+L%}w6I_7Ss)nh5lC%bwAQjNqhyN~YP2Ldn0|=Lpt;!q(a1cw@ksNnfu@Z2 zXPK-K7wvd}-(x&wMPm8;9@Th%hqU#ag_Usc}`)c4fe>T>mz zdQ)cw5ZqR8R?ny_>J3mitzJ`CYD%}Nuc>c?@Va^pwr{Ansu!b6HuVkIT2bEu$aev! z43;;lrv(7K0o&Jfh~?^8*aR1#^dBCF4`f(+Q4s}o`M(3X2_7z>rFtDAT|vlOHTSR> zju&KQjx9+C59}ACHue~#chY+R-vuFGk`-aGpa?HXf|M^RC-kX@6LA!jHV=blRtm+` z!VFv}y5K34>USIdbHy$K0h==N^$ErS$2UiovF_pUQh5CiZmXTb`Vp1cB~bV+{K?M& zxeu*hb(5Bki?Q>u&+NTbd+%p<;bXgS*Xh4vRCj*+Q|FA@G;`M-y5sa*%%9KS@r3T! z9qWrPo6R1}8Zoz8%KKJZ0Hr+hzyYg&$X)pKpPqD22I>2qbTVLi-)jY49OJ)?$d)}I zWK(>2^o7pax$lGxttnkBM$;wuufxkPjP{JE5R`!2FALqsG?sk~eFO06A~X~gA$MF_;*U~~`e^^FIHlFFr^fzt6Lu~0u{`G{%=|3K6bpiZ=*#+s;T2qF*tXXIh<5pMI51|L_<%#Vs30aUaN^$?)4 zh;jv|Xd~ZjE5o4}AmVL^hciN)hzaiRc#C%Mk2;okT$tB41!@t-z=2P%+m{4d;$QVj zwkRlAjGu9go{huO=vpo*vQ*7!MV5fQXfb*Um3(boE^AV$z|Nvgca>H0vZ|KKE24$X z@Y{}?G{r;CaXQ5>IfsH?^yx;lmxXa1L4lEY)fuES{B372aS^r#l?5 zF(#rK(>BF({Au?HP3xQPZ)p7tYO%M>R1JGEr%G$`CQE_Ju*(~AzNX2BUCrl|98(Qz zPEkbRAf_Q)ugPrFurXP!DH`(wOcg*RlaWZo3~V@+lB!8f231ys<7d(C4MOxU22ykp z^Dt~+f%LZW+38^C;DB8~ajeW%`J>*`H2rSbYpG8lN&>ORfrKr`<9RGsq#&0|azX52 zc|7ky;X;u^fjk)QMy^t&EK3z8mEq4YBlaQ+lP~NFlOdpD4cDzZvlZN4B_z?L0eEQ8 z(7PfZu)tKXE9w1#3{9>gL;>+{MSGdQ7KGGz*&EQm58kuPl@ZW?WEza2*Q4&u`tWpy z?}dg&3%Hcgo2@MwqZ>=IU)igCd^i-wPBJ`)Gm-MeT6s-nTX1qiFAjfBpE27Kc`@uv zhW=2s1A>6DgQ9xC=6IF8&9@@wp=UphT(k~W5LQLeDl)sy`x3MDql)@KHt;dB7z%v# zX8)T-WlzBtuf7ba7e#x6l^baD&pN~Vt*Lf6OTs6ZJi+hG2#>LZPMy_!ma>BwSfRKTJPW0K({6Udld|H{r&U@)FPhbzn@fX zW=qU*Hu=10bmpqniChha?;zM2Ufec^hqOO+D~x&hUz5H%-VrWefg#1%CERqb=hoyx z31(GCwE~@Y4V6~tSfc~ykU4$K*JpODVXf9QwhcDyCJMaT4q<6ro4ww7ZoN{dDe`0N z4(MQ#5cM;VKU*oK_X&URR4j6zEPqRucge^uIlW7!f9i6-IevHTZuYyb*=^g8j-dYA Psdd`@izP&(rsKZ>Auh_J diff --git a/app/__pycache__/schemas.cpython-314.pyc b/app/__pycache__/schemas.cpython-314.pyc index 8dd3dc72d54bff1f38fdcbbfc87d6599aa6bb238..7da303d553ff9dec31675697a689e06aedffeea6 100644 GIT binary patch delta 808 zcmZ8e&rcIk5T3U`y4zB=UE1yz1lxkBRbo^~jM3;pO{@_l3vw~Iv}FmE((3MOA|}S7 zF`h`kcOm`@9zC0Q^jf0_i3dD+Gp=dGlX2cI8skgeyqRym*_m(V{mHjywc}cE4Dftu zjNPflx3sJwyEMh`krLrZ^$Q)3`5-~;E>78-PZgtsm?zJ3`vNth7(VuB>(_h{2>FcO4faVrLTc=n zF>-8)r*pG_*VEBpLK}AVRQt-Fj0sP`%^>P7v0S{LJ(czIPT0OhsDxJzBnW|oh$T`5 zC8!n&BqaYx@LF|RqR#3RmqA{za$k-$RFyr0$079NC8n zkH-d(vwx7*$)x8I7LcZL8WxPYQFM0H>T;{GV%HYk%RJN;o7L8;)1Wr3(YiC`P9y4; zxvB8)vR7@oPEiaec{6~P>di8{n9M)=U%(05E;kxYTBeR|JIM9;?7|sq`ub!zV?EuG z(mtP_AAwA^BbmO!0~L~~&Do6y9VzceIS>O}N6GKuejF01t?`Z8_Ms2j!HzWQ_X07A zuR&a2zy3XelyDIS7G$RsLkWdc_!`gD5p=l;ddh}9wc(ozLxpx}w=mkix`+S!yG>4= zDkfRQoQ!YryEmOqoGbe6t6mvlz3VCg0>O(!P_q dWhA-Mfod6nxVUPvo=gSTHwI2d`65xE007eA5d{DM diff --git a/app/documents.py b/app/documents.py index c96d3bf..3da713e 100644 --- a/app/documents.py +++ b/app/documents.py @@ -8,13 +8,18 @@ from app.database import get_db from app.models import User, Document, Chunk -from app.schemas import DocumentResponse, DocumentUploadResponse, DocumentUpdateRequest, ChunkResponse +from app.schemas import ( + DocumentResponse, DocumentUploadResponse, DocumentUpdateRequest, + ChunkResponse, SearchRequest, SearchResultItem, +) from app.auth import get_current_user from app.document_processor import process_document from app.qdrant_client import delete_chunks_by_document +from app.search import hybrid_search router = APIRouter(prefix="/documents", tags=["documents"]) + UPLOAD_DIR = "uploads/raw" os.makedirs(UPLOAD_DIR, exist_ok=True) @@ -207,3 +212,24 @@ async def delete_document( await db.commit() return {"status": "deleted"} + + +@router.post("/search", response_model=list[SearchResultItem]) +async def search_documents( + request: SearchRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Гибридный поиск по документам пользователя. + Комбинирует семантический поиск (Qdrant) и поиск по ключевым словам (PostgreSQL). + """ + if not request.query.strip(): + raise HTTPException(status_code=400, detail="Query must not be empty") + results = await hybrid_search( + query=request.query, + current_user=current_user, + db=db, + top_k=request.top_k, + ) + return results diff --git a/app/schemas.py b/app/schemas.py index 36c6da4..4c54761 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -87,3 +87,22 @@ class ChunkResponse(BaseModel): class Config: from_attributes = True + + +# ────────────────────────────────────────── +# Поиск +# ────────────────────────────────────────── + +class SearchRequest(BaseModel): + query: str + top_k: int = 10 + + +class SearchResultItem(BaseModel): + chunk_id: str + document_id: str + document_title: str + extension: str | None = None + text: str + keywords: list[str] | None = None + score: float diff --git a/app/search.py b/app/search.py index b2f7039..ac51384 100644 --- a/app/search.py +++ b/app/search.py @@ -1,8 +1,142 @@ -from app.qdrant_client import semantic_search, delete_chunks_by_document +""" +app/search.py — Гибридный поиск: семантический (Qdrant) + по ключевым словам (PostgreSQL). +""" +import logging +from typing import List +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy import or_, cast, String + +from app.models import User, Document, Chunk +from app.qdrant_client import semantic_search from app.embeddings import get_embedding +from app.schemas import SearchResultItem + +logger = logging.getLogger(__name__) + + +def _get_accessible_doc_ids(documents: list[Document]) -> set[str]: + return {str(doc.id) for doc in documents} + -async def search_by_query(query: str, top_k: int = 10): +async def hybrid_search( + query: str, + current_user: User, + db: AsyncSession, + top_k: int = 10, +) -> List[SearchResultItem]: + """ + Гибридный поиск: + 1. Получаем список документов, доступных пользователю. + 2. Семантический поиск в Qdrant по вектору запроса. + 3. Поиск по ключевым словам в PostgreSQL по чанкам этих документов. + 4. Объединяем, дедублицируем, сортируем по score и возвращаем top_k. + """ + + # ── 1. Список доступных документов ─────────────────────────────────── + doc_query = select(Document) + if current_user.role != "admin": + doc_query = doc_query.where( + or_( + Document.uploader_id == current_user.id, + Document.is_available_to.has_key(str(current_user.id)) + ) + ) + doc_result = await db.execute(doc_query) + accessible_docs = doc_result.scalars().all() + accessible_doc_ids = _get_accessible_doc_ids(accessible_docs) + doc_lookup = {str(doc.id): doc for doc in accessible_docs} + + if not accessible_doc_ids: + return [] + + # ── 2. Семантический поиск в Qdrant ────────────────────────────────── query_embedding = get_embedding(query) - results = semantic_search(query_embedding, top_k) - # results уже содержат payload с document_id, text, keywords и т.д. - return results \ No newline at end of file + qdrant_results = semantic_search(query_embedding, top_k=top_k * 3) + + # Фильтруем по доступным документам + semantic_hits: dict[str, dict] = {} + for hit in qdrant_results: + doc_id = hit.get("document_id") + chunk_id = str(hit.get("id", "")) + if doc_id in accessible_doc_ids: + semantic_hits[chunk_id] = { + "chunk_id": chunk_id, + "document_id": doc_id, + "text": hit.get("text", ""), + "keywords": hit.get("keywords", []), + "score": float(hit.get("score", 0.0)), + } + + # ── 3. Поиск по ключевым словам в PostgreSQL ───────────────────────── + # Извлекаем слова из запроса (≥3 символа) для простого keyword-поиска + query_words = [w.lower() for w in query.split() if len(w) >= 3] + + keyword_hits: dict[str, dict] = {} + if query_words: + # Ищем чанки, у которых хотя бы одно ключевое слово содержит слово из запроса + chunk_query = select(Chunk).where( + Chunk.document_id.in_([ + # UUID-объекты для корректного сравнения + doc.id for doc in accessible_docs + ]) + ) + chunk_result = await db.execute(chunk_query) + all_chunks = chunk_result.scalars().all() + + for chunk in all_chunks: + chunk_kw_list = chunk.keywords or [] + chunk_kw_lower = [kw.lower() for kw in chunk_kw_list] + chunk_text_lower = (chunk.text or "").lower() + + # Считаем количество совпадений + matches = 0 + for word in query_words: + if any(word in kw for kw in chunk_kw_lower): + matches += 1 + elif word in chunk_text_lower: + matches += 0.5 # частичное совпадение в тексте + + if matches > 0: + kw_score = matches / len(query_words) # нормализуем в [0..1] + chunk_id = str(chunk.id) + keyword_hits[chunk_id] = { + "chunk_id": chunk_id, + "document_id": str(chunk.document_id), + "text": chunk.text or "", + "keywords": chunk.keywords or [], + "score": kw_score * 0.5, # масштабируем, чтобы не перекрыть семантику + } + + # ── 4. Объединение результатов ──────────────────────────────────────── + merged: dict[str, dict] = {} + + for chunk_id, hit in semantic_hits.items(): + merged[chunk_id] = dict(hit) + + for chunk_id, hit in keyword_hits.items(): + if chunk_id in merged: + # Если чанк найден обоими методами — суммируем score + merged[chunk_id]["score"] += hit["score"] + else: + merged[chunk_id] = dict(hit) + + # ── 5. Обогащаем заголовком и расширением документа ────────────────── + results: List[SearchResultItem] = [] + for hit in merged.values(): + doc = doc_lookup.get(hit["document_id"]) + if doc is None: + continue + results.append(SearchResultItem( + chunk_id=hit["chunk_id"], + document_id=hit["document_id"], + document_title=doc.title, + extension=doc.extension, + text=hit["text"], + keywords=hit["keywords"], + score=round(hit["score"], 4), + )) + + # Сортируем по убыванию релевантности и обрезаем до top_k + results.sort(key=lambda r: r.score, reverse=True) + return results[:top_k] \ No newline at end of file diff --git a/front-end/app.js b/front-end/app.js index 6eb47bf..45e9988 100644 --- a/front-end/app.js +++ b/front-end/app.js @@ -5,17 +5,22 @@ let state = { isAuthenticated: false, isLoginMode: true, user: null, - documents: [] + documents: [], + activeTab: 'dashboard', }; // DOM Elements const views = { auth: document.getElementById('auth-view'), - dashboard: document.getElementById('dashboard-view') + dashboard: document.getElementById('dashboard-view'), + search: document.getElementById('search-view'), }; const navActions = document.getElementById('nav-actions'); +const tabNav = document.getElementById('tab-nav'); -// --- Utility: Toasts --- +// ───────────────────────────────────────────────────────────── +// Utility: Toasts +// ───────────────────────────────────────────────────────────── function showToast(message, type = 'success') { const container = document.getElementById('toast-container'); const toast = document.createElement('div'); @@ -28,24 +33,24 @@ function showToast(message, type = 'success') { }, 3000); } -// --- Utility: API Fetch with Interceptor --- +// ───────────────────────────────────────────────────────────── +// Utility: API Fetch with Token Interceptor +// ───────────────────────────────────────────────────────────── async function apiFetch(endpoint, options = {}) { const token = localStorage.getItem('access_token'); const headers = { ...options.headers }; - + if (token) { headers['Authorization'] = `Bearer ${token}`; } - // Do not set Content-Type if body is FormData if (!(options.body instanceof FormData) && !headers['Content-Type']) { headers['Content-Type'] = 'application/json'; } try { let response = await fetch(`${API_URL}${endpoint}`, { ...options, headers }); - - // Handle token expiration / 401 + if (response.status === 401 && token) { const refreshed = await refreshToken(); if (refreshed) { @@ -53,13 +58,13 @@ async function apiFetch(endpoint, options = {}) { response = await fetch(`${API_URL}${endpoint}`, { ...options, headers }); } else { logout(); - throw new Error("Session expired. Please login again."); + throw new Error('Session expired. Please login again.'); } } if (!response.ok) { const err = await response.json().catch(() => ({ detail: response.statusText })); - throw new Error(err.detail || "API Error"); + throw new Error(err.detail || 'API Error'); } return await response.json(); @@ -69,7 +74,9 @@ async function apiFetch(endpoint, options = {}) { } } -// --- Auth Functions --- +// ───────────────────────────────────────────────────────────── +// Auth Functions +// ───────────────────────────────────────────────────────────── async function refreshToken() { const refresh = localStorage.getItem('refresh_token'); if (!refresh) return false; @@ -77,7 +84,7 @@ async function refreshToken() { const res = await fetch(`${API_URL}/auth/refresh`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ refresh_token: refresh }) + body: JSON.stringify({ refresh_token: refresh }), }); if (res.ok) { const data = await res.json(); @@ -85,7 +92,7 @@ async function refreshToken() { localStorage.setItem('refresh_token', data.refresh_token); return true; } - } catch(e) {} + } catch (e) {} return false; } @@ -94,6 +101,7 @@ function logout() { localStorage.removeItem('refresh_token'); state.isAuthenticated = false; state.user = null; + state.activeTab = 'dashboard'; render(); } @@ -102,61 +110,175 @@ async function checkAuth() { try { state.user = await apiFetch('/auth/me'); state.isAuthenticated = true; - } catch(e) { + } catch (e) { logout(); } } render(); } -// Make globally accessible for onclick attributes window.logout = logout; -// --- Document Functions --- +// ───────────────────────────────────────────────────────────── +// Tab Navigation +// ───────────────────────────────────────────────────────────── +function switchTab(tabName) { + state.activeTab = tabName; + + // Update button active states + document.querySelectorAll('.tab-btn').forEach(btn => { + btn.classList.toggle('active', btn.dataset.tab === tabName); + }); + + // Show/hide views + Object.entries(views).forEach(([name, el]) => { + el.classList.toggle('active', name === tabName); + }); + + if (tabName === 'dashboard') { + fetchDocuments(); + } +} + +// ───────────────────────────────────────────────────────────── +// Document Functions +// ───────────────────────────────────────────────────────────── async function fetchDocuments() { try { state.documents = await apiFetch('/documents'); renderDocuments(); - } catch(e) { console.error(e); } + } catch (e) { console.error(e); } } async function uploadFile(file) { const formData = new FormData(); formData.append('file', file); - + showToast('Uploading...', 'success'); try { await apiFetch('/documents/upload', { method: 'POST', - body: formData + body: formData, }); showToast('File uploaded successfully!'); fetchDocuments(); - } catch(e) { + } catch (e) { console.error(e); } } async function deleteDocument(id) { - if(!confirm("Are you sure you want to delete this document?")) return; + if (!confirm('Are you sure you want to delete this document?')) return; try { await apiFetch(`/documents/${id}`, { method: 'DELETE' }); showToast('Document deleted'); fetchDocuments(); - } catch(e) { console.error(e); } + } catch (e) { console.error(e); } } -// Make globally accessible for onclick attributes window.deleteDocument = deleteDocument; -// --- Event Listeners --- +// ───────────────────────────────────────────────────────────── +// Search Functions +// ───────────────────────────────────────────────────────────── + +/** + * Highlights query words in text by wrapping them in tags. + */ +function highlightText(text, query) { + if (!query) return escapeHtml(text); + const words = query.trim().split(/\s+/).filter(w => w.length >= 3); + if (!words.length) return escapeHtml(text); + + let escaped = escapeHtml(text); + words.forEach(word => { + const re = new RegExp(`(${word})`, 'gi'); + escaped = escaped.replace(re, '$1'); + }); + return escaped; +} + +function escapeHtml(str) { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function renderSearchResults(results, query) { + const container = document.getElementById('search-results'); + const stateEl = document.getElementById('search-state'); + + if (!results || results.length === 0) { + stateEl.style.display = 'block'; + stateEl.innerText = '🔍 No results found. Try different keywords.'; + container.innerHTML = ''; + return; + } + + stateEl.style.display = 'none'; + + container.innerHTML = results.map((r, i) => { + const ext = r.extension ? r.extension.toUpperCase() : '?'; + const scorePercent = Math.min(100, Math.round(r.score * 100)); + const keywords = (r.keywords || []).slice(0, 8); + const snippet = r.text.length > 400 ? r.text.slice(0, 400) + '…' : r.text; + + return ` +
+
+
+ ${ext} + ${escapeHtml(r.document_title)} +
+
+ ⚡ ${scorePercent}% match +
+
+
${highlightText(snippet, query)}
+ ${keywords.length ? ` +
+ ${keywords.map(kw => `${escapeHtml(kw)}`).join('')} +
` : ''} +
`; + }).join(''); +} + +async function performSearch(query) { + const stateEl = document.getElementById('search-state'); + const container = document.getElementById('search-results'); + + stateEl.style.display = 'block'; + stateEl.innerHTML = '  Searching...'; + container.innerHTML = ''; + + try { + const results = await apiFetch('/documents/search', { + method: 'POST', + body: JSON.stringify({ query, top_k: 15 }), + }); + renderSearchResults(results, query); + } catch (e) { + stateEl.style.display = 'none'; + } +} + +// ───────────────────────────────────────────────────────────── +// Event Listeners +// ───────────────────────────────────────────────────────────── + +// Auth form toggle document.getElementById('auth-toggle').addEventListener('click', () => { state.isLoginMode = !state.isLoginMode; document.getElementById('auth-title').innerText = state.isLoginMode ? 'Login' : 'Register'; document.getElementById('auth-submit').innerText = state.isLoginMode ? 'Login' : 'Create Account'; - document.getElementById('auth-toggle').innerText = state.isLoginMode ? "Don't have an account? Register" : "Already have an account? Login"; + document.getElementById('auth-toggle').innerText = state.isLoginMode + ? "Don't have an account? Register" + : 'Already have an account? Login'; }); +// Auth form submit document.getElementById('auth-form').addEventListener('submit', async (e) => { e.preventDefault(); const btn = document.getElementById('auth-submit'); @@ -168,36 +290,34 @@ document.getElementById('auth-form').addEventListener('submit', async (e) => { try { if (state.isLoginMode) { - // Login uses OAuth2 Form const fd = new FormData(); fd.append('username', username); fd.append('password', password); const data = await fetch(`${API_URL}/auth/login`, { method: 'POST', body: fd }).then(async r => { - if(!r.ok) throw new Error((await r.json()).detail); + if (!r.ok) throw new Error((await r.json()).detail); return r.json(); }); localStorage.setItem('access_token', data.access_token); localStorage.setItem('refresh_token', data.refresh_token); } else { - // Register await fetch(`${API_URL}/auth/register`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ username, password }) + body: JSON.stringify({ username, password }), }).then(async r => { - if(!r.ok) throw new Error((await r.json()).detail); + if (!r.ok) throw new Error((await r.json()).detail); }); - showToast("Registered successfully. Please login."); + showToast('Registered successfully. Please login.'); state.isLoginMode = true; render(); btn.innerHTML = 'Login'; btn.disabled = false; return; } - + await checkAuth(); showToast(`Welcome, ${state.user.username}!`); - } catch(e) { + } catch (e) { showToast(e.message, 'error'); } finally { btn.innerHTML = state.isLoginMode ? 'Login' : 'Create Account'; @@ -205,6 +325,19 @@ document.getElementById('auth-form').addEventListener('submit', async (e) => { } }); +// Tab navigation +document.querySelectorAll('.tab-btn').forEach(btn => { + btn.addEventListener('click', () => switchTab(btn.dataset.tab)); +}); + +// Search form submit +document.getElementById('search-form').addEventListener('submit', (e) => { + e.preventDefault(); + const query = document.getElementById('search-input').value.trim(); + if (!query) return; + performSearch(query); +}); + // Drag & Drop const dropArea = document.getElementById('uploadArea'); const fileInput = document.getElementById('fileInput'); @@ -224,33 +357,36 @@ function preventDefaults(e) { e.preventDefault(); e.stopPropagation(); } }); dropArea.addEventListener('drop', (e) => { - const dt = e.dataTransfer; - const files = dt.files; - if(files.length) uploadFile(files[0]); + const files = e.dataTransfer.files; + if (files.length) uploadFile(files[0]); }); fileInput.addEventListener('change', (e) => { - if(e.target.files.length) uploadFile(e.target.files[0]); + if (e.target.files.length) uploadFile(e.target.files[0]); }); -// --- Render --- +// ───────────────────────────────────────────────────────────── +// Render +// ───────────────────────────────────────────────────────────── function formatBytes(bytes, decimals = 2) { if (!+bytes) return '0 Bytes'; - const k = 1024, dm = decimals < 0 ? 0 : decimals, sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'], i = Math.floor(Math.log(bytes) / Math.log(k)); + const k = 1024, dm = decimals < 0 ? 0 : decimals; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`; } function renderDocuments() { const grid = document.getElementById('docs-grid'); - if(state.documents.length === 0) { + if (state.documents.length === 0) { grid.innerHTML = '

No documents found.

'; return; } - + grid.innerHTML = state.documents.map(doc => `
-
${doc.title || 'Untitled'}
+
${escapeHtml(doc.title || 'Untitled')}
${doc.extension ? doc.extension.toUpperCase() : 'UNKNOWN'}
@@ -258,8 +394,9 @@ function renderDocuments() { Date: ${new Date(doc.upload_date).toLocaleDateString()}
- ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) ? - `` : ''} + ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) + ? `` + : ''}
`).join(''); @@ -268,7 +405,8 @@ function renderDocuments() { function render() { if (state.isAuthenticated) { views.auth.classList.remove('active'); - views.dashboard.classList.add('active'); + tabNav.style.display = 'flex'; + navActions.innerHTML = ` ${state.user.role.toUpperCase()} @@ -276,10 +414,12 @@ function render() { ${state.user.username} `; - fetchDocuments(); + + switchTab(state.activeTab); } else { - views.dashboard.classList.remove('active'); + Object.values(views).forEach(v => v.classList.remove('active')); views.auth.classList.add('active'); + tabNav.style.display = 'none'; navActions.innerHTML = ''; } } diff --git a/front-end/index.html b/front-end/index.html index cb02dec..e7c948b 100644 --- a/front-end/index.html +++ b/front-end/index.html @@ -11,6 +11,10 @@
+ @@ -45,7 +49,7 @@

Your Documents

📄
-

Drag & Drop files here

+

Drag & Drop files here

Supports PDF, DOCX, PPTX, XLSX, TXT, Images

@@ -54,6 +58,27 @@

Drag & Drop files here

+ + +
+
+

Search Documents

+
+ + + + +
+
diff --git a/front-end/style.css b/front-end/style.css index 1605244..2a67478 100644 --- a/front-end/style.css +++ b/front-end/style.css @@ -259,6 +259,161 @@ header { background: rgba(255, 255, 255, 0.1); } +/* ───────────────────────────────────────── + Navigation Tabs +───────────────────────────────────────── */ +.tab-nav { + display: flex; + gap: 0.25rem; + background: rgba(255, 255, 255, 0.05); + border: 1px solid var(--glass-border); + border-radius: 10px; + padding: 0.25rem; +} + +.tab-btn { + padding: 0.4rem 1rem; + border-radius: 8px; + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + border: none; + background: transparent; + color: var(--text-muted); + transition: all 0.2s ease; +} + +.tab-btn:hover { + color: var(--text-main); + background: rgba(255, 255, 255, 0.05); +} + +.tab-btn.active { + background: var(--primary); + color: white; +} + +/* ───────────────────────────────────────── + Search Bar +───────────────────────────────────────── */ +.search-bar { + display: flex; + gap: 0.75rem; + align-items: center; + padding: 0.75rem 1rem; + margin-bottom: 2rem; +} + +.search-input { + flex: 1; + background: transparent; + border: none; + outline: none; + color: var(--text-main); + font-size: 1rem; + font-family: 'Inter', sans-serif; +} + +.search-input::placeholder { + color: var(--text-muted); +} + +.search-btn { + white-space: nowrap; + flex-shrink: 0; +} + +/* ───────────────────────────────────────── + Search State (loading / empty) +───────────────────────────────────────── */ +.search-state { + text-align: center; + padding: 2rem; + color: var(--text-muted); + font-size: 0.95rem; +} + +/* ───────────────────────────────────────── + Search Results +───────────────────────────────────────── */ +.search-results { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.result-card { + padding: 1.25rem 1.5rem; + display: flex; + flex-direction: column; + gap: 0.75rem; + animation: fadeIn 0.3s ease forwards; +} + +.result-header { + display: flex; + justify-content: space-between; + align-items: center; + gap: 1rem; +} + +.result-source { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; + color: var(--text-muted); +} + +.result-source .doc-name { + font-weight: 600; + color: #818cf8; +} + +.score-badge { + display: inline-flex; + align-items: center; + gap: 0.3rem; + padding: 0.2rem 0.6rem; + border-radius: 999px; + font-size: 0.7rem; + font-weight: 600; + background: rgba(99, 102, 241, 0.15); + border: 1px solid rgba(99, 102, 241, 0.3); + color: #818cf8; + white-space: nowrap; +} + +.result-text { + font-size: 0.9rem; + line-height: 1.65; + color: var(--text-main); + opacity: 0.9; +} + +.result-text mark { + background: rgba(99, 102, 241, 0.3); + color: var(--text-main); + border-radius: 3px; + padding: 0 2px; +} + +.result-keywords { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; +} + +.kw-tag { + padding: 0.2rem 0.6rem; + border-radius: 999px; + font-size: 0.7rem; + font-weight: 500; + background: rgba(16, 185, 129, 0.1); + border: 1px solid rgba(16, 185, 129, 0.2); + color: #6ee7b7; +} + /* Toasts */ #toast-container { position: fixed; From 303e27ac16c94696734033b3edded8b3e2f3eecb Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sat, 13 Jun 2026 01:51:42 +0400 Subject: [PATCH 20/34] Add AI assistent? --- app/__pycache__/main.cpython-314.pyc | Bin 4276 -> 4362 bytes app/__pycache__/schemas.cpython-314.pyc | Bin 7368 -> 7713 bytes app/chat.py | 112 ++++++++++++++++++++++ app/main.py | 2 + app/schemas.py | 8 ++ front-end/app.js | 121 ++++++++++++++++++++++++ front-end/index.html | 28 ++++++ front-end/style.css | 85 +++++++++++++++++ 8 files changed, 356 insertions(+) create mode 100644 app/chat.py diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc index a8f1591c92d8b748797f1582a6cd758517832474..ffcb7cb9bcc6b8dd45d112e6cd9d3cdb0c8af8df 100644 GIT binary patch delta 185 zcmdm@*rmj$&Bx2d00i4>bh4EBC-O-!CTvvqWfX{G2om;G@Q^7{lxK*Q3~-%X!KldS zI(ZhOqOvPUJt~M*G+~5@fK|Is-ovQRsshyKJ^2HpIivez4JLg?wapPs(>NI;H^1l3 zV`NO29LKwYQGGKzpCKbZM`A&NUUEia$t`XWIoW{Uohu4xAjr^So5_v*>zVH|XiYW{ KkeO^JPzC^Ht|*lN delta 131 zcmeBD+M>v(&Bx2d00b5}I$1aPCh|!z8f{efWn_t!3~-#>!l=mTIC&Y6be?R;EnB$T$V^@&kO2VxsUUR# diff --git a/app/__pycache__/schemas.cpython-314.pyc b/app/__pycache__/schemas.cpython-314.pyc index 7da303d553ff9dec31675697a689e06aedffeea6..97b8b941f76e2676d7a5347a430bc0bbca6ffaee 100644 GIT binary patch delta 215 zcmX?MxzL7Bn~#@^0SIQ-=wz+poyaG_XtYsXno&lSL6IRyN}M4`I+oFd8Ki=Np+q1^ zMwvlVcC!a#rqtxi>|&Escmx@XCztUkvT{3TB$iCRFD^T|io=K{1E_lPMHXdN39v9D ztB7bZP^g$6Nc?DLXkxg@&DZGliFtAruZnyLP_PI@fRq%efe2BUke{aD11W{OyrYbl-Z~*&BzucrOcoyy*Z9CQ;OM7Q(*ERnQ&z> rpt>Sa5FrF4ZgJS;=BJeAq}mmg0=bMpTs&!Vwd@40Zw%~=f?$OJQwbH# diff --git a/app/chat.py b/app/chat.py new file mode 100644 index 0000000..58f0340 --- /dev/null +++ b/app/chat.py @@ -0,0 +1,112 @@ +import os +import json +import logging +import httpx +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.database import get_db +from app.models import User +from app.schemas import ChatRequest +from app.auth import get_current_user +from app.search import hybrid_search + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/chat", tags=["chat"]) + +OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://localhost:11434") +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3") + +async def generate_chat_response(query: str, current_user: User, db: AsyncSession): + # 1. Поиск контекста + try: + search_results = await hybrid_search( + query=query, + current_user=current_user, + db=db, + top_k=5 + ) + except Exception as e: + logger.error(f"Error during search: {e}") + yield json.dumps({"type": "error", "content": "Failed to retrieve context."}) + "\n" + return + + # Отправка источников первым пакетом + sources = [ + { + "document_id": r.document_id, + "document_title": r.document_title, + "score": r.score + } for r in search_results + ] + # Дедупликация источников для UI + unique_sources = list({s["document_id"]: s for s in sources}.values()) + + yield json.dumps({"type": "sources", "sources": unique_sources}) + "\n" + + # 2. Формирование промпта + if not search_results: + context_text = "Нет релевантных документов для этого запроса." + else: + context_text = "\n\n".join( + f"--- Документ: {r.document_title} ---\n{r.text}" + for r in search_results + ) + + system_prompt = ( + "Вы — корпоративный ИИ-ассистент. Ваша задача — отвечать на вопросы пользователя, " + "основываясь ТОЛЬКО на предоставленном контексте из корпоративных документов. " + "Если в контексте нет ответа на вопрос, честно скажите, что не знаете. " + "Не придумывайте информацию. Отвечайте на русском языке.\n\n" + f"КОНТЕКСТ:\n{context_text}" + ) + + payload = { + "model": OLLAMA_MODEL, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": query} + ], + "stream": True + } + + # 3. Отправка запроса в Ollama и стриминг + try: + async with httpx.AsyncClient() as client: + async with client.stream("POST", f"{OLLAMA_HOST}/api/chat", json=payload, timeout=60.0) as response: + if response.status_code != 200: + error_text = await response.aread() + yield json.dumps({"type": "error", "content": f"Ollama error: {response.status_code} {error_text.decode()}"}) + "\n" + return + + async for chunk in response.aiter_lines(): + if chunk: + try: + data = json.loads(chunk) + if "message" in data and "content" in data["message"]: + yield json.dumps({ + "type": "content", + "content": data["message"]["content"] + }) + "\n" + except json.JSONDecodeError: + logger.error(f"Failed to parse Ollama chunk: {chunk}") + except httpx.RequestError as e: + logger.error(f"Error communicating with Ollama: {e}") + yield json.dumps({"type": "error", "content": "Ошибка связи с Ollama. Проверьте, запущен ли сервер."}) + "\n" + + +@router.post("") +async def chat_endpoint( + request: ChatRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + if not request.query.strip(): + raise HTTPException(status_code=400, detail="Query cannot be empty.") + + return StreamingResponse( + generate_chat_response(request.query, current_user, db), + media_type="application/x-ndjson" + ) diff --git a/app/main.py b/app/main.py index 3f24fd5..c0b6d41 100644 --- a/app/main.py +++ b/app/main.py @@ -75,9 +75,11 @@ async def lifespan(app: FastAPI): from app.auth import router as auth_router # noqa: E402 from app.documents import router as docs_router +from app.chat import router as chat_router app.include_router(auth_router) app.include_router(docs_router) +app.include_router(chat_router) # ────────────────────────────────────────── diff --git a/app/schemas.py b/app/schemas.py index 4c54761..0f37415 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -106,3 +106,11 @@ class SearchResultItem(BaseModel): text: str keywords: list[str] | None = None score: float + + +# ────────────────────────────────────────── +# Чат (RAG) +# ────────────────────────────────────────── + +class ChatRequest(BaseModel): + query: str diff --git a/front-end/app.js b/front-end/app.js index 45e9988..b63a778 100644 --- a/front-end/app.js +++ b/front-end/app.js @@ -14,6 +14,7 @@ const views = { auth: document.getElementById('auth-view'), dashboard: document.getElementById('dashboard-view'), search: document.getElementById('search-view'), + chat: document.getElementById('chat-view'), }; const navActions = document.getElementById('nav-actions'); const tabNav = document.getElementById('tab-nav'); @@ -264,10 +265,130 @@ async function performSearch(query) { } } +// ───────────────────────────────────────────────────────────── +// Chat Functions +// ───────────────────────────────────────────────────────────── +function appendMessage(role, content) { + const messagesContainer = document.getElementById('chat-messages'); + const msgDiv = document.createElement('div'); + msgDiv.className = `chat-message ${role}`; + + const contentDiv = document.createElement('div'); + contentDiv.className = 'message-content'; + // Using innerHTML to allow basic formatting and sources + contentDiv.innerHTML = content.replace(/\n/g, '
'); + + msgDiv.appendChild(contentDiv); + messagesContainer.appendChild(msgDiv); + messagesContainer.scrollTop = messagesContainer.scrollHeight; + + return contentDiv; +} + +async function performChat(query) { + appendMessage('user', query); + + const submitBtn = document.getElementById('chat-submit-btn'); + const inputField = document.getElementById('chat-input'); + + submitBtn.disabled = true; + inputField.disabled = true; + + // Add a placeholder message for the assistant + const assistantMsgContent = appendMessage('assistant', ' Думаю...'); + + try { + const token = localStorage.getItem('access_token'); + const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }; + + const response = await fetch(`${API_URL}/chat`, { + method: 'POST', + headers: headers, + body: JSON.stringify({ query: query }) + }); + + if (response.status === 401) { + const refreshed = await refreshToken(); + if (!refreshed) { + logout(); + throw new Error('Session expired'); + } + // Retry once + headers['Authorization'] = `Bearer ${localStorage.getItem('access_token')}`; + // For simplicity, we just throw here and let the user click again + // but normally we would await fetch again. + } + + if (!response.ok) { + throw new Error(`API Error: ${response.status}`); + } + + // Streaming logic + const reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8"); + + let fullText = ""; + let sourcesHtml = ""; + assistantMsgContent.innerHTML = ""; // clear spinner + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + const chunkText = decoder.decode(value, { stream: true }); + const lines = chunkText.split('\n').filter(l => l.trim() !== ''); + + for (const line of lines) { + try { + const data = JSON.parse(line); + + if (data.type === 'sources') { + if (data.sources && data.sources.length > 0) { + sourcesHtml = `
+ 📚 Источники:
+ ${data.sources.map(s => `📄 ${escapeHtml(s.document_title)}`).join(' ')} +
`; + assistantMsgContent.innerHTML = sourcesHtml + fullText; + } + } else if (data.type === 'content') { + fullText += escapeHtml(data.content); + assistantMsgContent.innerHTML = sourcesHtml + fullText.replace(/\n/g, '
'); + document.getElementById('chat-messages').scrollTop = document.getElementById('chat-messages').scrollHeight; + } else if (data.type === 'error') { + fullText += `
[Ошибка: ${escapeHtml(data.content)}]`; + assistantMsgContent.innerHTML = sourcesHtml + fullText.replace(/\n/g, '
'); + } + } catch (e) { + console.error("Parse error chunk:", line, e); + } + } + } + + } catch (e) { + assistantMsgContent.innerHTML = `Error: ${e.message}`; + } finally { + submitBtn.disabled = false; + inputField.disabled = false; + inputField.focus(); + } +} + // ───────────────────────────────────────────────────────────── // Event Listeners // ───────────────────────────────────────────────────────────── +// Chat form submit +document.getElementById('chat-form').addEventListener('submit', (e) => { + e.preventDefault(); + const query = document.getElementById('chat-input').value.trim(); + if (!query) return; + document.getElementById('chat-input').value = ''; + performChat(query); +}); + // Auth form toggle document.getElementById('auth-toggle').addEventListener('click', () => { state.isLoginMode = !state.isLoginMode; diff --git a/front-end/index.html b/front-end/index.html index e7c948b..8c9ac94 100644 --- a/front-end/index.html +++ b/front-end/index.html @@ -14,6 +14,7 @@ + + +
+
+

AI Chat Assistant

+
+ +
+
+
+
Здравствуйте! Я ваш ИИ-ассистент. Задайте мне вопрос по загруженным корпоративным документам.
+
+
+ +
+ + +
+
+
diff --git a/front-end/style.css b/front-end/style.css index 2a67478..81923e4 100644 --- a/front-end/style.css +++ b/front-end/style.css @@ -123,6 +123,12 @@ header { display: block; } +.chat-view.active { + display: flex; + flex-direction: column; + gap: 0; +} + @keyframes fadeIn { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: translateY(0); } @@ -451,8 +457,87 @@ header { border-radius: 50%; border-top-color: white; animation: spin 0.8s ease infinite; + display: inline-block; } @keyframes spin { to { transform: rotate(360deg); } } + +/* ───────────────────────────────────────── + AI Chat +───────────────────────────────────────── */ +.chat-container { + display: flex; + flex-direction: column; + flex: 1; + min-height: 400px; + max-height: 600px; +} + +.chat-messages { + flex: 1; + overflow-y: auto; + padding: 1.5rem; + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +.chat-message { + display: flex; + flex-direction: column; + max-width: 85%; +} + +.chat-message.user { + align-self: flex-end; +} + +.chat-message.assistant { + align-self: flex-start; +} + +.message-content { + padding: 1rem 1.25rem; + border-radius: 12px; + line-height: 1.5; + font-size: 0.95rem; +} + +.chat-message.user .message-content { + background: var(--primary); + color: white; + border-bottom-right-radius: 4px; +} + +.chat-message.assistant .message-content { + background: rgba(255, 255, 255, 0.1); + color: var(--text-main); + border-bottom-left-radius: 4px; + border: 1px solid var(--glass-border); +} + +.chat-input-area { + display: flex; + gap: 0.75rem; + padding: 1rem; + border-top: 1px solid var(--glass-border); + background: rgba(0, 0, 0, 0.2); + border-bottom-left-radius: 16px; + border-bottom-right-radius: 16px; +} + +.sources-box { + margin-bottom: 0.75rem; + padding: 0.75rem; + background: rgba(0, 0, 0, 0.15); + border-radius: 8px; + border: 1px dashed var(--glass-border); + font-size: 0.8rem; +} + +.sources-box .source-title { + color: #818cf8; + margin-right: 0.5rem; +} From 56a42c5e8d2cb642c52d949242c990a75c8d1663 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sun, 14 Jun 2026 00:17:41 +0400 Subject: [PATCH 21/34] Add: alembic support --- README.md | 63 ++++++-- alembic.ini | 149 ++++++++++++++++++ alembic/README | 1 + alembic/env.py | 97 ++++++++++++ alembic/script.py.mako | 28 ++++ .../versions/3cad579bf02c_initial_schema.py | 79 ++++++++++ app/__pycache__/main.cpython-314.pyc | Bin 4362 -> 3786 bytes app/main.py | 12 +- 8 files changed, 411 insertions(+), 18 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/3cad579bf02c_initial_schema.py diff --git a/README.md b/README.md index 09e895d..af844a5 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,56 @@ sudo pacman -Syu --needed --noconfirm build-essential pkgconf clang llvm \ pip install -r ./requirements.txt ``` ## Как запускать? (работа только в .venv окружении) -- Билдим rust либу - ```bash - cd parser - maturin develop - ``` -- Запускаем python - ```bash - docker-compose up -d postgres solr - uvicorn app.main:app --reload - ``` + +### 1. Установка зависимостей +```bash +pip install -r requirements.txt +``` + +### 2. Запуск инфраструктуры (PostgreSQL + Qdrant) +```bash +docker-compose up -d postgres qdrant +``` + +### 3. Применение миграций базы данных (Alembic) +```bash +# При первом развёртывании или после `git pull` с новыми миграциями: +alembic upgrade head +``` + +### 4. Запуск приложения +```bash +# Загрузить переменные окружения и запустить сервер: +export $(grep -v '^#' .env | xargs) +uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +``` + +Фронтенд доступен по адресу: http://localhost:8000/ui/ + +--- + +## Работа с миграциями (Alembic) + +Alembic управляет схемой базы данных. Это позволяет всей команде синхронизировать изменения в структуре БД. + +### Создать новую миграцию после изменения `app/models.py` +```bash +alembic revision --autogenerate -m "описание_изменений" +``` + +### Применить все ожидающие миграции +```bash +alembic upgrade head +``` + +### Откатить последнюю миграцию +```bash +alembic downgrade -1 +``` + +### Проверить текущее состояние БД +```bash +alembic current # текущая версия +alembic check # есть ли незаписанные изменения +alembic history # история миграций +``` diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..29f1c75 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..e0d0858 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..480014f --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,97 @@ +import os +import asyncio +from logging.config import fileConfig +from dotenv import load_dotenv + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from alembic import context + +# Загружаем переменные из .env (нужно для команд `alembic` из терминала) +load_dotenv() + +# Alembic Config object +config = context.config + +# Настройка логирования из alembic.ini +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# ────────────────────────────────────────────────────────────── +# Подключаем все модели, чтобы autogenerate видел изменения +# ────────────────────────────────────────────────────────────── +from app.database import Base # noqa: E402 +import app.models # noqa: F401, E402 — важно: просто импортируем, чтобы модели зарегистрировались + +target_metadata = Base.metadata + +# ────────────────────────────────────────────────────────────── +# Берём DATABASE_URL из переменной окружения. +# Alembic не умеет работать с asyncpg напрямую, поэтому +# заменяем драйвер: asyncpg → psycopg2 (синхронный) для offline, +# и используем async_engine_from_config для online. +# ────────────────────────────────────────────────────────────── +def get_url() -> str: + url = os.environ.get("DATABASE_URL", "") + if not url: + raise RuntimeError("DATABASE_URL environment variable is not set") + # Для async-движка оставляем asyncpg; alembic использует его через run_sync + return url + + +def run_migrations_offline() -> None: + """Offline mode: генерирует SQL-скрипт без подключения к БД.""" + # В offline-режиме используем синхронный URL (psycopg2) + url = get_url().replace("postgresql+asyncpg://", "postgresql://") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """Online mode: подключается к БД и применяет миграции.""" + # Переопределяем URL из env для async engine + configuration = config.get_section(config.config_ini_section, {}) + configuration["sqlalchemy.url"] = get_url() + + connectable = async_engine_from_config( + configuration, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Online mode entry point.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/3cad579bf02c_initial_schema.py b/alembic/versions/3cad579bf02c_initial_schema.py new file mode 100644 index 0000000..80f17f2 --- /dev/null +++ b/alembic/versions/3cad579bf02c_initial_schema.py @@ -0,0 +1,79 @@ +"""initial_schema + +Revision ID: 3cad579bf02c +Revises: +Create Date: 2026-06-14 00:11:48.436585 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = '3cad579bf02c' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create initial schema: users, documents, chunks + indexes.""" + + op.create_table( + 'users', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column('username', sa.String(), nullable=False, unique=True), + sa.Column('hashed_password', sa.String(), nullable=False), + sa.Column('role', sa.String(), nullable=True, server_default='user'), + ) + + op.create_table( + 'documents', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column('title', sa.String(), nullable=False), + sa.Column('author', sa.String(), nullable=True), + sa.Column('uploader_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('users.id'), nullable=True), + sa.Column('upload_date', sa.DateTime(), nullable=True), + sa.Column('last_edited', sa.DateTime(), nullable=True), + sa.Column('extension', sa.String(), nullable=True), + sa.Column('size_bytes', sa.BigInteger(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('file_path', sa.String(), nullable=True), + sa.Column('is_available_to', postgresql.JSONB(), nullable=True), + ) + + op.create_table( + 'chunks', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column('document_id', postgresql.UUID(as_uuid=True), + sa.ForeignKey('documents.id', ondelete='CASCADE'), nullable=True), + sa.Column('chunk_index', sa.Integer(), nullable=True), + sa.Column('text', sa.String(), nullable=True), + sa.Column('keywords', postgresql.JSONB(), nullable=True), + sa.Column('language', sa.String(), nullable=True), + sa.Column('start_char', sa.Integer(), nullable=True), + sa.Column('end_char', sa.Integer(), nullable=True), + ) + + # Индексы + op.create_index('idx_chunks_document_id', 'chunks', ['document_id']) + op.create_index( + 'idx_chunks_keywords_gin', + 'chunks', + ['keywords'], + postgresql_using='gin', + postgresql_ops={'keywords': 'jsonb_ops'}, + ) + + +def downgrade() -> None: + """Drop all tables and indexes.""" + op.drop_index('idx_chunks_keywords_gin', table_name='chunks') + op.drop_index('idx_chunks_document_id', table_name='chunks') + op.drop_table('chunks') + op.drop_table('documents') + op.drop_table('users') diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc index ffcb7cb9bcc6b8dd45d112e6cd9d3cdb0c8af8df..d09b0c3fcacd7b6ad19c1bdb0e95280c1bfabba3 100644 GIT binary patch delta 1050 zcmah{O=uHA6rS1LY&L(pX=|Fb&0o_RTbs75M*M5lCKOsIB`kUn6GPi<+eo?_CMyUE ziimfgJ$Pt(5cK9nSV0fsL8zc8coKZGZLtdCz`Xh9y>H(4-eYd1U&iHO zIpjyMCZCO*e;~JH6_;9jE8R(S4(Za2G99+V(;aq45|z2)K)z-8L-u|T0B6r_~4bE>0UeL z%5}}e0VlR_5^xG+)nZ{Q*^Z+e+t&RjBk^BGiZR^g-6T`2G5Qn7JpCX#d4up%`j#C5iTV;=TMSE{OK;kc%T7q9?f7OSIyRaGp7O&ufZ) z!W8L*q*lecwY+Q)N2r-gmgAecxM(!&nq@j3gAj`l9}o$f>JU$4I2>WoGED~?O&)HU zREmU|SE|>RjAmorBr&+Eu$xc7e1ai;k?4kG;0Rsv9lvrLoY!z6hp+b)1oQ~!)6!!e z(33=d(6i>n`2lH75*Q8!z`Yj5`HZlp?q*oi;9Wb70k;P*aM~Fc4~hlsYmef*Cb!4@ z3?Jo~9hZQ)E(8i1zpki-EWf^2@j#08tcSTYzL2AL{ZDa>jt5@!WT7(B$&A~OKrG89 zBXB=WPx)?%eX4%3tJ z&8Q>KHyWeH>Vi?N*39MR)=23GIUDF^dTe4uraejmKI9SQ%rH}RJge3P({|I7ziN}l pSFP%Wx^bMGrq7iT{v%4#&q`K-U;2$lT;vPKK5do`hTjRSqhIo|%&!0d delta 1465 zcmZ8hOKclO7@pae*UxqAB({_InOE~_NEH_lB&dxY5~W4P6-Yrgu48W#BYSPluBB21 zC-gu>g=EYeN~IQ%0Er9YSRrxA!$ImPRgnS|DSKZ{U0;` zKeM;IS3TA0$0E#)h$IiU-*reGala4-wT#)H-+s9ME*1s_OrXdVZEnAk#xbF5M4)*64!oq`tpEAREd!(DcRM9Tog?D!RK#!qt} z8HtV-5ohpCZl)2(7dQ-)=x!ObD(?r0MlzO9c<~}XZhUM2XKoSs2G{rr=bh(RFUUc% z{)kA0NQv@O8%;g{5}mDx0?76P&;v?L!qVbmv`R-toWp^-U#%jFc4%vX;&g zZ{3tZa$1v9>3m)n^2Oz48R=Y3xlq&%nWCb|8BI6H7v)S@laYbC8lx98a!Jb-6`hw6 zDk4Oig3Lq>u}GpO9PwOJbSSH&wO}9+Ky8#3^SKMMT1qP@NS1CQ>Jy^=B=_Q>0PLOB z)hnyjO11J{<&*aBZH{W;-NNvGDf7 zo_**$Zup+S^Uwa|^jw|(V0z!#yYIx~hF3d+#PElI9jcWpmA~#=@H>X-wYwzo<eqU7J zw^1s$x!`_ zNIusAByM#El5lmdfdwJzH`XmMD%t8*6UAMoI5hRm}PTOV`Jn%G^QClPi{JYjYv zdag@Sq?l0y=j1fXtOQQQ=F263m&I!^h@tV@lM|#8h`x|!lQ>i?YUl)6FA`Dd`&|dW zSlZvuemK83IJ0Mt>~Rs~#7~S5;c@)HbjH!>t*|1e^V$kJP3+I(m(5Y=#8=HrmOUD-m=ltynnWIsdQW}EXmnyPFYqDt`PpzGHRyppm9NvgmsG#4&%p`#Syw8 zoi7y2iq^=1j%-_ezEI4T^YS!$4Zmd_V}A$z_)F`kME*1nSip4LW9W}(68~ZSnWKmO E2f$27B>(^b diff --git a/app/main.py b/app/main.py index c0b6d41..637077e 100644 --- a/app/main.py +++ b/app/main.py @@ -10,7 +10,7 @@ from fastapi.staticfiles import StaticFiles from sqlalchemy import text -from app.database import engine, Base +from app.database import engine from app.embeddings import load_model, get_embedding_dimension logging.basicConfig(level=logging.INFO) @@ -23,12 +23,8 @@ @asynccontextmanager async def lifespan(app: FastAPI): - # 1. Создаём таблицы, если их нет (без DROP — данные сохраняются) - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - logger.info("Database tables ensured") - - # 2. Проверяем подключение к БД + # 1. Проверяем подключение к БД + # Таблицами управляет Alembic — запустите `alembic upgrade head` перед стартом try: async with engine.connect() as conn: await conn.execute(text("SELECT 1")) @@ -36,7 +32,7 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error(f"Database connection failed: {e}") - # 3. Загружаем модель эмбеддингов + # 2. Загружаем модель эмбеддингов load_model() logger.info(f"Embedding model loaded, dimension: {get_embedding_dimension()}") From 2ea49d14c296b14dd8e3181674622c48fc8b990f Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sun, 14 Jun 2026 02:19:32 +0400 Subject: [PATCH 22/34] Implement: Groups --- ...bfec_add_groups_and_available_to_groups.py | 51 ++ app/__pycache__/auth.cpython-314.pyc | Bin 14087 -> 16189 bytes app/__pycache__/documents.cpython-314.pyc | Bin 13481 -> 20222 bytes app/__pycache__/main.cpython-314.pyc | Bin 3786 -> 3876 bytes app/__pycache__/models.cpython-314.pyc | Bin 2879 -> 4028 bytes app/__pycache__/schemas.cpython-314.pyc | Bin 7713 -> 10117 bytes app/auth.py | 37 + app/documents.py | 268 ++++++-- app/groups.py | 190 ++++++ app/main.py | 2 + app/models.py | 41 +- app/schemas.py | 37 +- app/search.py | 33 +- front-end/app.js | 634 ++++++++++++++++-- front-end/index.html | 47 +- front-end/style.css | 434 ++++++++++++ patch_app_js.py | 465 +++++++++++++ patch_query.py | 7 + patch_ui.py | 26 + test_api.py | 8 + test_api_2.py | 19 + test_api_3.py | 10 + 22 files changed, 2165 insertions(+), 144 deletions(-) create mode 100644 alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py create mode 100644 app/groups.py create mode 100644 patch_app_js.py create mode 100644 patch_query.py create mode 100644 patch_ui.py create mode 100644 test_api.py create mode 100644 test_api_2.py create mode 100644 test_api_3.py diff --git a/alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py b/alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py new file mode 100644 index 0000000..80dcb42 --- /dev/null +++ b/alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py @@ -0,0 +1,51 @@ +"""add_groups_and_available_to_groups + +Revision ID: d3bf9fb0bfec +Revises: 3cad579bf02c +Create Date: 2026-06-14 01:50:32.654089 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'd3bf9fb0bfec' +down_revision: Union[str, Sequence[str], None] = '3cad579bf02c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('groups', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('created_by', sa.UUID(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['users.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('user_groups', + sa.Column('user_id', sa.UUID(), nullable=False), + sa.Column('group_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['group_id'], ['groups.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id', 'group_id') + ) + op.add_column('documents', sa.Column('available_to_groups', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('documents', 'available_to_groups') + op.drop_table('user_groups') + op.drop_table('groups') + # ### end Alembic commands ### diff --git a/app/__pycache__/auth.cpython-314.pyc b/app/__pycache__/auth.cpython-314.pyc index f62eab06cd5ae49298e02db1cc59277ea6ac9557..7875512923b7c0631b2f314bb524b94ea0fc13a7 100644 GIT binary patch delta 4784 zcmahMYjBgt@jJaOS+;EXZDUC`k}bb6PHYSq94xSdov5D;g~o~^B*hlUl5?K~fu@mp zG(a8=^wI)tQZmV;18t^Z;?U+%9_=JQ`lAzROa}4|ZD>n>v_CSYQ1Y1Rw7VzCHl%TG z?EUuk_V#x7cK22XBj=v1Ys@q11sFZa=Ix!$g77z-WQR6`-2D7#-E?8UC1$Y*4+|n0 zu*R$&h5Npv*NXFNI!Kx;R_h_|{-f85Orl|qX3x;rDq>JOvXe)MF{jQ`ZxgbFr%7>+ zb0Oikq*&dMUHvC9svENtlfqcswbvsHJEElvL{;vu{37)IQ~B$ z`>rC-?+`0W1+lVJ2%BOpVin*^INmB&18(7Xn`i~x#_?rh4dAsLZx@#WZs+)NPp(Z6 z9b#Q)))O6`9I#FA-i|u4|0)W#_j+_d>0o6_`-)DEcV;zRPQ(_F)rzvhB-Z88g0wbq zSql-{I~5t71S_@jlRVU0TnMjEGpTNNNRV-}J9kAwtxwOw(Ur z+9U<(aL`Bn<6=;Xhhx)TO}9erge~3EKhwDgS<0^G)GVn7f?O~bjCm)0lC&#I1Kx3; zG)_@=H>=gJQ8fU@+R9h6SM^n-mR;1_TkJq`DyReZs_+mbaYMoZVB;=2A)FfsD!mbqjV9Pq(*#|(@(%_?T(l}j@a&l=uuAqnL z00I?)`w^fnvRQPk7hS{sUe^})favn}^==)egSecFpqZU7ENg!N(FXy{>kkvT$WMd5 zSkRj}&>M^H2uA2uSlYsL=7$&hTb*7@JDj_kt_8lu%=EKab1nDx>+E9DuQg~Kb+IRk zmuy^y=nw+bOjbfPQUhS86?7D-ZUE_^08b-_IS}+m13|CP?+;2+M$rJQ_}TU1by@E} zgnK;z7E>z%Y8A0FCHbUb`eMl@@`x9Aapg86=EMD(wD-84Tad~_+e;0C4P1 z#Ie56y{yOHSP?@s4*@!w#u0FyU2MO7X~Kf@r3h>YupY`vUjTC7ADx&8#VCe{tO-ZQ z#)4E0Tf?)$|$Mz%Sr&#E{B0k<@@Wpk8FcSVx0nAt?sez+hn-{v|yTgntNA z!tAueArf*3+%dEgluvsEj+n@L1UrUr^vItWsYj30)khSj5boHHUDMh9tr8c?{} zB9wyxzu47&#(vWThOib}`Sk6kA1erD&$ZSW7MR2?v=%3(P}VV&WulmE^aO(WMAhO{ zhu|j&7B}9EEb|s?7Tl{Fmw2%78~!OwB@H}%>D$d*Z{8i*FVU00b$dGAwu3bC%*QnN z#_XzOhW08jyuSVA@9f88AV7r7+o-{Go(R%cfm8u>3BAapMw`x zO)KWR|N2>~25KTHz z$zu=5x-lA!PkKWE%I)NNoT+X+-@bC=iM#p+8(P!_vZsX41PiITf#9lEe`w_UduEk@ zU(YKz()Zl|6 z9W>;HyLJge0E$ADFp8exLsW=LQI&1c#tC%a>H$T+a#CdxMsu=UqnMC(`W5nGxMpW~H-IywOjS-c*m6`R#0=};iRkeU6gN|spuuL3*ox+@*%hArw>bV0JY+hhUb(TEm4$#?#9rLhm zvydx@x^`vOGuba3qjcA1H_m_IFnp)(8Ve6&rsj%O?yQOn7$0uW3eSEw2x885xcSG) zlyXq57bdlvwQeW-w7x>&Zp6<>tzZ>KO<5rtA&`$2LnO?{i$>HgjF*tGQn^G35v?%T zbB{7aocj2OuwS1%nLM9-JGD3YE^D)tCX#29@3dHxuK@AA)U(NVljp6;w*Ycc0s zMgJH)9DMwB>IG}+K=N(mOYKWdS(6ul8S^{hn+RI#2L=Z=x}5rK0$Pzj`DXHLYASgy zwI6moYfbH2a7^l8^1Wo@Z1Nn?-n0U^0Gwyx@Uw8jQ6S^OJJ#fMYAUrK&bp9#M$cc> z5Q7k)7(;XlfRo4u)WXY8mO@8hbx-L)WT!723RvgV!B@-|lZpKSXG!{T#Jo(%df((^ z%LMdwVTqnXnRAb`tcu4&0a>+a(?FlB-|P#=gDy&=@a%4nh9a^8)`(x$^0T4f@TdIe zC;>~ftPX`kJA%B}%R2ah0}5{2Evv&Jc*SL{Qkt2;S#!l`!vFlXh8_lPiFE^}7!JlQ#jB)KJ#=^s!j;uVk`snJ*W@pOm zJT;s$x6K%r&1!`l<7aw7mvbfG{QRE7dtO|6viU^w<%;H1Me`YwIAc3wKGS<@e5Rsv zCco>D>YBOuSohKHPI&`fpra`aOXR7;V%&yBu*VOtehJxpN z4)@F$DlZ!>DT8IkU`wiOpWiAKOcl2T!mY5JT9@ql)}=x*nJdmJk@n?{A|Y=*`9f*~ zKkZxPYHcPLN?iHE?}~eux|+%FyL;Q=^L`U?HLBikt^xdm{Bl>l@`HL6;LiJ)dU=$z zu=w&mHT?yMU>_`ZBq-KG`bz+E4m^&7>E=x5^`edS*n%6dotKrN zNQ`d8>9-N^{?CRB+Fd>x2}QlGM z+4KPNXy8qReot0SMx_|<`glXcTZ43)@+z{a!q67-0HUwpUVO;XA*Nde2@($nd*~m5 z27x6=aj5CwQbKMh*rkpZlZGh%EdaQgCy?B0Ii~6Sdp;!_W)%ujHmesDx`P#aD-QL) zJd$ka{>Z*^s$zz$x>i(nrPA_oxix89HdEgIacTAOoh)%1AJLmiL9zN@itKaxKT2IR A?*IS* delta 2829 zcmah~TWl0n7@jkGX}j%~?TxleTejVHw@WXSQfQ&ja%r*1tqcMxEX%SxElYQI+J9z& zVx&M�M@p!5f<3i%Dam=my_R;KlG}@NrBu#0NE^LBVKDNc{gZ-4?u*CjIt5|2h9T z-+!IYPo7H$TZ-K-3IEQktw&yOmZa}!k^6Jb2oKI&4rZN;rLa8Q$8Qysd&0ghX6+gE zVKzJ(NgH;p+pU~W96^`cIeyLKXly}K9Plj@8 zUL(6JmW2Jm8J?r%l2jvw>xcL9y9H(bg}J8}OG>5T4>$hzIetzqjx{RYN=aE%DJ4Bd zt5Sno3O((x^ujIY9;=C1Ny@C(hDi^H`=b?RvvaFpco!vQ)yTjeX6T<$At;?}$ zc6Ha*=U6?v>b7?}dT}*LCs*}_>y)0{4b}^}cr0g1KQdlNjFYTuCVIwNh54n>`fQ^k z!I+(2a1LdEaITUaSQb)GHc&9gn4c$I4J^nnx+)#ZNzFfW^|KKF$+e#~@S(z{_D0l( zWN5-4(|$!2A~uGT_~WRUoBjM=BK^o{Cc^Ae_MESs0BAeHfX~iGcR^n zjg5^TgH}??u!0O%BIp#EKtNn)E;y<}Vq`Kh9@X_@8pL5WZuaxC*1T=av<(_i zW!mFWBMQWDTE;~pW(c11G2FtZ-9Z-0-f??n*2?cKsAX%p%Tv3U95KscphgWf5{<=F zU5^;rn3{&2xU+?C@`SC|gRlk_(1$SJLwk5tiS>|=@q?uomTn-cjRqM!VY)bQ?mz z=Yt@hDDyg=$JHKwsqDZ%V&<@%vVtbU1^sw*K3Agrc)1_1x#O8c>MRv<&-5y92>WIuYK!u-XwrO_V1WZp?-G zMLM@1VLon2-dH)qjo&Ee{9)zv`U7MrIPNCtAYJh|t-Yibaf^~fwBQiJ9G`Ehyp9pn zB>AJN0pzox`pb1xae})k#}A33!>XpX05?Gi!hA%}@}1tLSa_$syJRQ%3Q=y>?At7& zq0yQ`0vQC;iPe!w8m2v?tDw`=4q##_wY2NOSVm$Q05xqy=mbon`fT=O?IN~{Seivo z>{6=1gD@YxtNg3FCZzXgT{ZKas8}CQCDJ~t4Sb+ZWD=mpF?VBhSCTLDwKl(mq8I`* zU0d=vtti>U89Z{8zv4T?#;9+5XZ!Y$7M}!i8fQQ$T}dIFkKU*JQa#c8tp1^`<|J7h zCwQ66Y$9l4jKCECqN&BJ(_S$FWd#fZ3M?Gug@MMYIuiW^J_4G1rY#zeo6eY)N+k@S zvN4@WZFE!x)1F9=XfTDUkOR(>fymY|l1M9<=nj-blB^u%gF2ef4b07D-1r3lbr%lI zAEi^${p?2pg|SyLBEvl*U3R=bl?w-by`LJ6ulf0A3g?_2-LmH}StSUL5KQp4 zmZs4Ehk?$}P_?44lE2ilq-H0rX=lL$L>-AJS{Wt(;2J@QKvZ1*8gwj^RQuot>M(YazJlLe)8lq9 Y`GJIRzgTjWWb@bl!d$<}7)oLLH$A>!4*&oF diff --git a/app/__pycache__/documents.cpython-314.pyc b/app/__pycache__/documents.cpython-314.pyc index eeb3b4961a489ab619cd42139062b86febc3d93b..9c316b54f810acacb846a0eb72f542c45824416c 100644 GIT binary patch literal 20222 zcmdsfX>c3YnP4~2xK9$`eKk)J5)YZABwD9QN+f7f6bwO-Y?Ed{5+Wgy1l--Uj!Y6t z&L-5@6G^r+lE>ays>zJy+SG^Qr%Q~M;m_w_rzcfap?kE^T}BL&X(+xmSS_fyn=!w*FjKH`xg2t^Cv~HcCE5kghZoS(e7znKHHo8rMiNH*^*=-Rl1lDw0 z-8RAIwhMN5mXPJn7P3jaw%Z{%0Ef2QDL6@(BjiA+>&|uO33()szB}JtAQZR@g+h0c zP~&&CPIeFB$U1;>?0-Q(S)dO{&;Z!*V^5zeec?*=%0~)Bku!WXW2ej~|{j0xJ(8lT{ z7W~RnzMA#kB4}YTws4$oNi<4mg0_w-Elh>pl7W_i(z?P$F{zf{s}_~F6ly6;t7Xky zYGDUT7mdih@st6~L=bmbQs=~}48l~&7!)s(k>f)+O3WyDv0&K|4++;wTVH+i*{l&j%W91xdI zGPHTRz2PY#$Vs}b!ANj&gqJLCpU<=H;BYV^gu;`OaaUwK92n^gjRz$r;fJ{i$<`hi z-apEPrzS^y0e(O4q9rpXdxLx=Jjn;~yNy3IIlME-^MDAxwR~_qI4rmrNgd_}B_=RA z0{(D-7bNXYfeTHJ_F=BkpfETx1i6@9JTy$dH9R~u5u6kxO~=^O=$B@j0pZk$?a#eS9hibz2D9WG>6jV1OGQQ&R29CqFeVbPK@=XrdjP zI6TF1(5gXf4aP;d@Gx|Ba3s-Oh$|R@4hzA-VQe=)ICN-nqGjyR5EmL5mrBw`xZvKGH#E2bWz$7nB|Ivu^L{>0HqQC@XF5E0pVnSQMTH!`qnG@oKVNw2g{ zGA|wjNi{MgX}O><#Z9^x&H-7uZ1j*mNz70}#)EJ0FQp3~`IV70}2LqFnVF9LdaFDA9q%8b+ z7kJauZK{;cx{luWY8NoLR7HXhdPy)h5Fm*+Au~0M!=*SJf+s18xj73R4p0?T8DQ&I z2AA4Nd6EOB6rm_(SElm$X-{%6m*;EUqfCgGuB0Gua=ey9;Cn zH_#6;4^ks#^2qq9X4shG$4XE-*h!4bO zcoIhDJi9nDL1ajpORgO-0A3ip=r^JFPl-RSThiKOALM3tZ@t?XABV29X>E{eSk(74 zY~o9InJ(xanIlL(k5l#oR{UXf8fOgty@s>-dZvEet3IEk3E-GZ%;ePgcpsrI$~woR zvKf9i9{_J7Egz3QsuPa<$Fp{-JP@1ia7!MfrZr_$PYy+qg-%scxLjYMk?1=z47*D$ zsTx*G>e0{$R|DmFtha`p=IS7n3>!y-lfi=#ZnIQ7JQf_@pQNpWfng-X4I9V9!+~*r zb3-!c2E>94@t;1016{Ok|MBE3e_|}2+Z4@h`q0+&#^fEIoc%?mbf-%%sd<3|c7sq{ zU^zk~k_vtd7Y1I)mNt1D?r_O5h^#@1L4h*7qz+6Ta%o5(INZBjEoLuxaIyT$dl!<)7}F(^W#mf|j!8#x2!ROZA+gI&SC@4L#@bgu^o(T0cV1>(YyYclcuWq@}9xLmL7j(^7 zubHe*HP7KZ#jM+pHy>~Q+vHc=wEePa`xQst?@jjW2CC3a-{FC#QBSrNZp)=Fmgb_r zCLjE_ta<3yIM83e4*a*93c$Z)qPIDjOJ*y=MNagaO2B`|t^t1uKcxf2xcYAH1@E)? zsZ7iP@K{M!0C=nha7p-}&F2jF4gr+JWeM5M4@yZMYo}#g77o20eww( zehRrA1*Z}MC8w59a+P1@v89)Va)~HAwM#vvYM`c6op6Lj)csjXUcdS=YVq_bHT0=^ z^5nyAN)>@A1q$6Vr78mq5p{?hejFa7E`y{C9t;kn7>2Fq06itq!;+rJIC$)64vYo4 zAh#70^m~KC@K|7cT%K-*#L7nsaNj7UNNO&8fE;AWBZS+Gh|wc!;0XZm2%LB1oLYZ! z{Z+m7=`SDu%CWD+ZPig*bxdCq*Vje$b>DmR`vb2G#OvFm_3al%qxIWk`o5cLl~a40 zQdzarop5+mt$caQ*)4OGEiu!YXl2Xkj;A|Ls86(gTl=jZ_>7v?h|HSnI>>$JhL$p~ zy~9^Q)koXd_EP#{1>K&;W9lquG{F8h z$@||vl$5gv1r)R;W-rR(8X+W%2m2&#U`iMZ1DP5L^21z+2<_Y^=m;T5DS870$bhkU z^pKU463*^*WNe4*{L|o}IQ1Y6JN>#GT+OXsP?LyHjLroOfo{PR+fF$DZvBEz{?=0# z`-!T*dvL)Ze;cXnoD;&g3l~iCx0%W!UdcBZKoW03t2Lp zO&M~oI|wYFO4uF6A^%QJrCI1-LS9qSGD>UZ$$cF$G*A(vUxSnx-N>WJzxy>FB*JOV zn>rYm?8+G1DoyI7d&H ze9AyS%zmv>-^kBNKeV6Kk6o6;skRc>w~y}C8wGQsC%{PgE%(X~=O3_Y$m~U|5sP2v z0huw~d70J&Fy^g93IhgGc97Spz^Tx$UnGTTmYm%-zZK4Br6;f&9{)sqWLgqoR390dg|p`fG*~yTCYGUsatLv=%96xtbUN!W#?I> zu}#Los!4bLETv5cfIdgEBM!eqnV}Ju%)P?{vS_;V>XmYy^7OP6wc8G|YsC2(t*TsJ zz8}wb;{48JYkuyP*OI~gY=YY zNOpLFL>YT3)E{zfG=6&uO-YVco6f~$dCNV|#i z8GNecv+B$8seKvOvCp|D-^ciJd`4vuWJ%s<#8LN|(_+mS>ou=SXfYy%KD}p+QgQM@jS1wZJh)lXT>p9{A~jpSz38O26YNwHLa zk|)zrBS5ZE)6}2jw7j%epndJJoI5l_v90S@z4m1TtAEa{d)Elm1N>U!#Twd4g>om zX-9$rC@wf`Q{Q2yQi3pxi6Rp4j}4OQz)Xv}0_A}FcCz!hPuBt1a~J&?5H1f`@I4<7(40<$j+$_(Q!Pw$Sl zt%F;;y@Usnm@rfY5`8d$Og0x1f|44LN`|BXgebB>gdZ7^w8PMe*Buh|MpcKJJjpEp4PF6P9WVSo0PT~F&lWuw|!KKC|-Ky6GN}0V=AdcU;ru&O0jNj@qcB_MGEGNBvC4e0I($*Gbp2 zt7bZ`*|LxE3o0gSfWGR?Kh^zA_vzv1`sSULap%gYbLF|_51ozIO_Z}?A(yJG78&O? zUE#cIW8Bpqb+un;o^y4@a&|>sU1!>Uv+ft`eslkGp%Z~qqbEoIJ_M07MlomCyeThk zDvp|p&*v?tH2Ibotpw zW*!s^x_;*1sC~sb^@sMl>pIF=x?rd5PFYN< z5N#D_RMYOO7TfXN$9BgpB~eSs>9Lq)#gWeGwrT#_s)iHlQ^u3V)6LJ?;`ZvOy?V~R zVs`6v_b2x3Q@WG7Q|6QA7a#de!MUDuo6ghcUFVCUu5ELB9u@mXV|&J8_Rw_qf=a9L z(^oB7CtRm}aZAOVrQ%FrzLnTxL$6rbMxLh;Ol zw{?`Y@)oo;f8LZkZ*|12Wumq0#KC!6&b*U7wd3TD`Rt;3N6Eai41jB^)<18%z+7AX zK(x5-!pN;0lU+OQUMQqAx^H$K>6~euW1RCw^KsWP*Hf$FOpeIpe9SoGOi7d}nJaC0 zv*Us(wy{^-6A8T!Ojes2GTx;fJhk=gO5TOpdtuee3gIPrxmW;?|8U84VCan~c_ z1CP!X_KVK`1vMuA&l^?>DES{sdhYz8pzBY!j4<3#45uWwa#k&F_KG|GqIb8ranD@t z!=n9RCs6E{_YVKRU}Ya|Gi(QKQPzqco$6w%1>m=|4)hE1dTN-r);Pg`+fWBF zZ?BYtS_s}VVbpsS&fZ$; zy|vEX9L@Vix;I<@zKKP6l^){XZ?;07U#JSuFE*mTq8|KTI>9u+{4%Er;9u#Adnz@* zs>J60szHN(b8)Xt^MOqb@CVs6VEZ7))LU)%fMo#Z2URr2SE~`OHEqvQf6!d8-K_q# zSp)E|vlvL_24Do1NYo5OAYs3W6y`6Hs2f!vx2lQUnmX&?{6`U;=&9}{X}$!R_+nDy zZHlEUuA_G~J9j}1;U{|Mbq$2e^-;)2JPE&>#n9}8K8o@{M7r}*#Fhrc2Y6B1gB?tB z{YYhqONr$QEmuO|C+Q`U2WpKLOQH!33a^z(R#4I^seY7OiStvQq#a3#1Nn+pINxRU zf^7{X9ws5XRe%Jr$XuIA(()vwt3~)3rC%T$RV8F24Pd&*NO({^mFD`91`?MVIZsle zQPQA48GDJ^xqhTg#AU>V>mto%lx=9GKmFxO_C+nqfTbTsPIk&jd#aU~ROnSHc$Hik zTWL>~P@pf+$_!Y(Xe*PgO)8u{8O9*f%1H6EnddAy9>8CsYf$?&JLolV2Z~ARNFv(I z`^=xbCADl-|5dw*@bteV{>g-#z@XgFTTi%bwahWYd2}B%_^|5H0hd$eDZAlYR=wgL zk#s!RjRMGzp_VntaLN`Q-B^Gh+#fu|Axp&J{v~WPhr5@sdXgF}LmVDU9L^+sP7cbd zT3zZ;g={Qcxe2(R7nk_IlPaEEvuVc29{z64_wr*EO|iV@cvka_7MOwKUp@BKn5`^stBl$z zW43BhUkwVp;>L4-^Zl>A^0f=9SW!njr(?SBx}I`Y+|;P^wZC_i&ve{UQ--X#zACD( zlC{L=c7CY8?~0=wtv@*n6yyW)ZE-x;8Z#Hi%@t8|#Z|rOs@WMgvr#j9ruXF?XLm%+ z&7YXO72ULJZudZ}^|5&CP_%VO9DY33 z%Ejz_q7Y_}J6od8mRVoSx&F0r!J6}MDPcg~x#j&DA;Ic6$K7}`wbB2#{29hKd3Lr>*yrSIHeU=*&`Qr7A_ z`~xsKN1NSd`Zc<13-d1<0J?Y|+|6VzHt7*wU+dN|Z&hqv4#&u7xceopHV(Bpg2`UvA^6ur(UoE31#B}w~H1F~56k;}6`dUkPKJUbE#N|AbIqLCh%fH_?w(j_L%g&lTH5lXJ+D9d z>Z2DrqiY_FmG;COy>Uw~>`+@y+*TX4)y8Zqr`^{~4p|&%anD)1MSb@tSv6-4p66m& z?Qv84ho*LtI=f@ex>eL~{n+G~-f=yPvTmoZ8!2-K*p*j96OOj2+KcFm46#{hiV!X@ zMt>6xmaB`K4FJETx1(P`gTG{-f&x_Dr~Lr@pWLT#M-OomC*RSNxo;eENyo`I98-f2 zvys}l$e(?_?K=-SdE{Dg`&Qug1uwaM0sh2Nv7mtvwl5fLbO`BjBO3scH!=gI6_?z| zPeW{CBdhmHZq6?8JPNn$-_ki5ltpm9m*wVRT(sjB8HOmqeJ7> zF4^p~2ZgOP7lb$>^khi$YbKN83$Sa>w4(5tES(_@N+;CPD6C{McO*+^Foi(rq=6qw zXAl^ITM{kLkft1fBXF3I-et%GB9b;CBEiT-v?(zo#e{lT^5 zKwQg|2YL;n4>5)rBKqxBpIIA@U&h)};}-!;enWCzfN&L4K0{(elRE(jW%=?|rU=F9 z;{XOwwWo4`g_KFjaV_BAzt%&L>F6$aT{wzK7d~IXAw&QYb zM=bA4(b|qP_g&)OzwhlQ#D@l>U4t)bUsRuN`;Hb8X53=lmuB1_r?PJ)*_UM4daZfG z*@G9H*o>-=mug{ADZBF6$`gBH#?m-bDl(<>$hq9OA32vhH(06)7#KZO+DqS&^1gKE z4|!X!YeD0Ag(l!7@BSd85(yTb5T=-aCrwtU8=I8-{f>RGVf?fu6OmcyM=kz zkPQj%7G=BFGVeCoz<-a1xR~Li-z8RjNG4EUQ5w51YRm}S}I)odw z=&v<_{|glj{t`9`83W{YZ~os9XSkDqnR^zzdubt6u-223NHth66$lfO z3P8&x6&+OaB=(~idh`FYqyj`Ed8|W!6K)BE!(?P|=r7FRFqs&fB*}S*5ZxrBl8GVl z6e#UXg~0Muze@VUw@LY`IA~RrB)z(s!X~_ ze^O_gf$*t3N@JDgKzMi3>NR+cjdX%oTEWhpWOcyKP1vf4CdI;db-s)}!+RY*4OqLA zEkhf+mx^e%XnGIG;S#I29|-OPgy5R~3W6&?u3AcP59g*#-HCLx60?U@>sc-cVHU-A$s+NScSBFgK3g1ftQBcUL?rQYwV{)~YPKHl{@ZT6V_zH1t(`NwKrqylzBqiQ=7s%p z%*upi@q{~OECHJ$*bvigjb`VGkZ58>hP|-@DchgG`fM)%BOk$@C4%+1HNI+7bk!zt z<>vE47qTv>#q6zd^R}pY8=Cqc+n<39YD%v#Id}L`7(yU?ouGGP=xy0Z_z*5{%Y)#p z`y0T2+r@6np)MI}0lst}-Kl3THDT&I2D;P8ykj&VoMY6X1uPN@xG40~ zeifu-5S+ctI#4vAgA;SBoyJ ztbJ&N3rq?we4RvIY%OlbPvE`Qx8N<(Yw(ilDQE*AcmwK!$6WAH>k0G1C>PxMP_L`O z$UO|5JDm3r+7;M|DPACYzo8&Y-k=b12Ob@K3o@k?MsUqI92wl-2Z|$-Fovl_XhFf2 z`yO~uW~umr(g|Ke=Mz`ZYZ8_VySyEe=NZraS14YNi*PppM8a@VX2#78dxr z!y{oo+U9I4qc7&tZ6(abJRQJ?vyvM1#>8Ej%=`5J9V@ySM@CVayJ+ooU|h9HRuo{e zMIlUVQGk|jQPi&hE6Usy{$sGBOO5BBLutMj7|bQga!{4ShE_qpz>U#&RhBQQDeaI+ zsoqh4wg?-rZlDWFcS`Az)ub&+Rxy*d%#)X18Z=`Wu_}z7f_<4bQRUU8wJn2&Yx%`f zuile1rl;BhmuO)q46fS?fd>qE)2hakUEY3o=Q*U9TPc;~q2^^bf$V^y zLedzE8H7?_Z>^59CwfFJ~h$MEk} zz{4dl;)jLcVf&tX#nx8u!)H4{PKgv@oCuDD0)xV#NKn4anV+CVgHUlK%#*tSBw5mp z!5vk&B{xHzBEf6gaGO3UUbHY+ z5wb_IP^D<9Jd+(au8JC0o$H7hSI3QOqsFx{W9xLsRY!5$vEs601*%5ZM{C!|92<`G zPIq53Ip>`XGlq{#%jc@s&Xuk`S$n>By637b>-d3V2jaG}sIBabKF&5p*`}DSdD{K2 zOvNXrym`3yU@41Q%H|!#@Q8Q5yzXOj=~>O0v6m;$PR45<5NjSduaB{93u;xC@s>$% z)-F(bjrK-8N?bR9okA;U2MjdIPOT_AbvA+Q)PnH+bmto8;(7zXZ!x9lS2lqEc6J&1 zP3=t(yjyKWe+>=(X5`TERrg*=y)QBOp9vFvr)Dwpq;I?!_ z0RJ>iC@jC8&03VWcMAT1iTn@~?!PAs$v1s~7ok#Dd$`x}a~pcN%Oo?L4h^6~58zhe zvM|bsQRLAgFUw?j!H7fsk^IWjgy}psE5LJTAtSfRIi&Mk0eVE5Lvjwc&LJy4_Tx}V zBju9H64vs;#C3bPDUZZm%EH9QiP$hWJRZV};7Hcx=E!U~hYUQZFUVj2;W+%zPXzYE zdBpJ?IX*UH90RA2K(2tplm!K@3nIy#BVwS3d7Bc5p@2Jw38JjVx)0mO8gzD7M3ryh$^k9|a~{D^A&h^o3qHN>cf-%PU^6kYQX)%+2)4)XurkbS!D-2QU|bA}DmTDX{Q zEt)pYQ>t&8j+$m3JmdT=Rr5zH{27=(w$t#~8LpbE^xrHxQZ)0hxMI_EQH=>1lG7tE6kaKZp#`63Ii+%V*${O)l|+7TBNdIQsfa4&1Er~ R%~T_ZF<7wthDs4={x?Z>mU93A delta 6049 zcmbVQ4QyN2b-wSBkN>~okN7K!k|>FiEJw0z$+i+pmL-#kC9||F8;N9w7U_sFEivys zI=0u0N;;rPlOne73Oj9wrC0~VL$JEZfHgy~)rQ@;#kz@;3o0+ll3*UNV8uX1#Igi(g{~(@n+W+QtW+Pl1uE;m zR$OS07ev!B66d03(JYr1(E_C^YK_#1bpZmrI%_4@12V6RHb#PCQ0Dc~rbx5cEc1qFOC%%)Ljf|w1WDW& zH+3^{b3a>Ir`Q^|h;1|GSd0!i?Dm8a`Z73 z6*DTRabKav2FXmIHV$ib)!H_7+*wcJ_4Ooc$ufP!u(8?>SkMt%&>`1M*Nm&nQ;{RR zXL-L%F&lS-Voz1^hTbhzy`$b4*Z*6S4>bAj)6`wnbf^JV=>1A741nIo`}A%Fy-D18 z@c&#PeVYw%JOWVNN3ZtB5;ZYJ~4rkC(>dfbv&$)l!t{hFKPBI%;ry} z=S7Jdnaj_geACV87`k2;)>*)u3V7GA(zPAUbVlW&&)AGKXN&n&6-$GkXs0z*eyavn zq5GJnpV|f4dl-8D{f55A!7QI4uiH;CL;J}!+Ygtu za#cs_9OuRFc*=UYY9LPc<-YGcS~kj66S38o%`$5tTF06d*>#xJO=fFbgDPH{Z@|0$ zhLUzUdN@rt$!Y0vM?a^V(#6#DsjXfGaP)1*cB8K9Bmpv|rLh6KbLt+^Bl`)_k_Ixx z&}$=h`nUBqg)*n7AMFd!*GKHl`s(1P0af^4P8r)^R(w*80FGqJ7;~d1>T4ho^TOmF z)i@JZ$&fK?wXF0Q%OgGos z&2jZbx0ka_F|iOW_WEtAC&53?FpG)7CWSlbX08ZlrH`wlX@p%j;ne`~yAe4fMagTj3`erjbl{RJ*E3Du6r4*{01^SzH5` zt7e=@8~t;ydxt7#+hfZx8B)=qCI>EVS7ey5Nn*M_J`XYEoFh7aI+@L+8V?_i?rS`; zz@JKr^tWEON+J@|Jwdnof(5O_W>S0$%*2O)Ncwy(yO2z!`9vlqY13!K^gQB+q!Tjd z(uw0|#k9b;Vn^+X3|!`1Qk>)Q=)l4CM#0k1oeA%pw5IKM`j62T`OF#H5%P z!hT7a&u3CziNm5#;)Gl_1Hc1!Ka&GIf)&$W``py-A7VP_VSk$nPlF$z&-lFs7qWFg z!cHF21Hv~Wsck}Dpu8%+9UIW^Dzlq6VF;l|kQ6XEzaQEqCLt+cyZkQfEU%t9A*p8< z=HUcHez)8uEooto37?n|vuR0F-LK$;*4B-O9&HFUKMn#fyuxq64j%(*-E{)CTV)L)q1HgRHx$714PnCCVI*V#CQg#te{mrpw$DSX! zt+cP`3@?XY2*21_QaYBD4w#&C&`&}?8nLqZ2Zldi+Y1Yh0Cy^=Z3Y>dghw7ZU1)m+JF=I>*nEJC~mM=x#6})!sNyl zd&J4zWSEFubCYdEzDWb}n;npMF*mmjo1u7zWugw{I|>~9&URnK%H6UmfWKvDV8AV> zKGLea6;#5oTOkJJfmQ`p!un{P;#QwKYE=B($N~TJIwdso+v%~USUp~nO{0U;RpJQ! zK}R6_7#aj42?h}h<5)woTIVb)puD58uW?ZF2r{!9pa(1=7=*q7@A^w%%O*gmMKTRs z<(aPatm0gbogy(M!m^nRJPlC&gbcJS#D%ZzwK+EMh>*>ZF&!f1P7>Ql#;Urp9Eg0b zC+_CHCuT*xMq5=3vl}1?*yu0S>W1o~swiHq!y=iO1E*zJy433C6jO>=J;n|eLJGvM zf^tgPOaN0_WK3ZtE}1%&nV$$7c|FX;<7JeT`~qA9?&-NSpXPD#JOUm+fCMo?QqLuY z#L4to9??xwr_ZEk^I}?33$w{=5^rBJn}yI{yQC8LR6fn0l~jBoE#$KzFW(^!2Di30 zHa7BIsK%7d2#~gHV2!|MLHH=V!X+T`UDFYx?GFc*4Zg+E6}7FTb`{mG%lmJun?JNT zF6l4mR}JP>i>qX5DOy@yGyKSW#ay)X%aZn@rM+Z%plC@y@E5kL(|?)xabme=blEbt z&MAzhbp>bDJ%y+f>1A6>$=I?uwqkK!GF>n&TN*1`mo2S}(G|UYNeNy> zOrg&NM62iadPkJZbw4x0Dz7($MwIlw!)L?jvXTmM1)!*Q6jzk|5H#@nkZd|}&jQp1 zJWh0CGh<&vvTCrr9DN~LRseqo9GZg?pafDFwVAGU_?e$kbLR*2@_=8N0B@v!JyK7X z2E0m8?xTsF^|W!PRt@m)BhwCgYoLKzv>N*OPA9vQq5m?_Vvs%IyXXm5)O4cDyS*1z zH4Efp3RmHrad;MCzA=@a&!kgfJr6!lHXVgeV|9SO(&ayrLRNMY^bpxGzyoksn|tR^ zVii_Ogo^FK_!Zk5n#m*$ZB9A zV+8mB!9yXRZ1!(;x;bu&ivKJ;3 z_m;|2+&Qt_ba1&oRbwE{zYR+ z<1K2u-ybdcwiJC^t~!gp{$J{3WKYp z4;1x*W&OcmNgpceL(BTMC1u+uep2@}X3YuxJ`?bkf=`b&qfd`Epid(YJ`Kh7uDxa` zZtP}8gUTC&&A{IbGNaA(jVy#lYWgBJS! zp81d-+`g4^JKCP;lo9BYCekv_e57=|t8Dm8I0zlVbYo^T-IxMRhkS4ZMK@YZNPF#L zD&q(%lP)ptC3FFf_Ug^Q`elb;E$5g;{N z0WZT7lpFBmfAf3&!wSd$cG!mtdqAP2PQlL%B18fEmpgV$x%mi^D5_DV&p?R?A*4;t)0 z?s|yDeV4E>^bU1B7}Rjc{$|ZWhadb^P`*KnFxRGYgcZC41)I^FmB^b%ANn_j>7o0F z{A1|6VI}|M=SE4KxR6VnoPcX5H;iDb46gVa9shD=%p&p?H>7_yv_*9R>hBWz{!pF` zZ>RtH@D{rF5m!M6QpqYjl}%=6=hCOncAvvX znnBoOKAV1s|5Ip!xIqL0sXu%mG0X?#Scx1fl4I|at?!eT_lf5{qJNKQ-y_Cw~ zSCMp4)A(+-tfEun{mt&irC`spyZ7DthRdgE;omDiGbsqWr_ACnPUWvN^sVuBGklG# zgS5ZQ0#|OOYvZ9PBz7plyF`vxn~#@^0SM+^(9L=`kynzjYomH7qd*)(kg%tshjfXOJVUHRfcxYIMnz?J zkU~@tD`&z85rL{Zfh^PL^UaXY`yL#H7!tzPX7ho}Dp#^CQkY zM#i|wf!r$?H8%g^c4idjN-QYQOE1bVEhxUl3!x{Q^15?{11$$RpxAeE8}BCOy9|1h bE%>zfZZI%kXAru`AT+`D3WMV0B)$v)*7q|s delta 138 zcmZ1?cS@F5n~#@^0SFv->SoQI$ScVhvr#>iktJ3lz30+4i_T*xXs zS)ECSQF*cellbIlCKagIWT2Sp<_%2o?2MtC1-SAU8Dl2bbFW}j+pNLk%s9D?*PbgB eXbB?_7u!re&%257!Q_Q};*;m_2~FO^mjM7nk|W{( diff --git a/app/__pycache__/models.cpython-314.pyc b/app/__pycache__/models.cpython-314.pyc index c74c2570c3c87b017e1bf8ce20c0ea50f63926d1..c9ed9aee15e677142de4f4df42c25dde5a153b6b 100644 GIT binary patch delta 2026 zcmb7FO>7%g5PoZ~?e%}0_$Nu5x+$fNljesuN?NNbTEw9TB&zzviJ4uejZl!lQoc7c z@6GJYH}hV;kA4^L8wmJ41b%<~H2BVAFCo97vh@qJgs0cJ)xPA>84vTs$s!T?S?`RG z`DXmgZ|aWOz)X+@P1!lyHWOl@I7xeYiRem)&h6KK;5({z(1f%P&@Fn#IV*D71-e)C ziT-h$Wr5wmIN#bupWu!@ZU5#o+VW)`5dz+!7!ljkp;3Fv3dzEzxdY5$G`Fk~(;5M5 zbelD5T4P}C+-8lX-4M7d9XmI&db`)r4Z>mlXRc##m$+N(8Ro=2TlvR{$h|-k@yGa; zvkA@+axznxT2K;#ewdrA8jh5l$tww)!Jj`rJ8d}T7Us`R!a*NZ@-kCPMQuqfgU(ON znqqh>nnI;Zv{Wf;hRamcT&m6)E?JW*6*c#lfJSXd%^9w#iG`_&>65T4l!`eeuP_Ay zdCOES$aGn{tSlQz!HgxTxod7KDtKlsN-T7?l&9B0sF5^boJg2$F`|FNy;n^KfCx`I zh#ef?qOwi&vyd2ox`R*^q+>YDq*AmUV$^N46BfgjQ!dJtJUdml8O};ky;4yOp;*c1 zk&ik@d+`vDj-c#8#c>+zHe3azkbxS(FYeZqEYt?&7=etOy)2bWIXQRh|vL54j~L83?U37;EkX@gb{@5ONfUN z8~_GilnXFhIYrA-wTweY1+;szQ~@q?Qf8UD@HFNM&0^VN>D}kMGJ8%gJ%H!N5il!@ zp<^hSH6g?aq@$_&h!qx{GEHT1nB96&6-HKSCul}=vE7Q{&Neg4(PWX zM;!D7#MWKTF^L`r$#7p(^NLiK*%I}m9#l0+zACHc#gJIZ=-95c9)lezpTMLMuoc=3 z{bT1B9kuz*?!D`g&j%ZP%;Ic9A9M9_Nf^#4{Y%&NH=k8~&6g`>D+Q&<@I9e;5oi>_ znz!ixE%qzn2-v~OY0_kV z{&ma&|4;m5cn#rrp_HrSl@k=F3nryXv>!qG)6Q|+uf%(w^gJMozmUbdWbpwRyH5`O zPI{jBNHBbl58dO156`@RW|Q}<99zw;k2a!x_5O_uw-Sy1)Sb-527d{+lB;Zex)Dp% zlN*>aG=`*55IBHYAb2V3(k^kaU(U{IASQ6& zfH%z?JQzHh5RYC>{1c2`OvV^uOiYXyKYH_^ZE-fxj=Xhvler|RQ{*cBj*Jigu&GAQ2 zvxV}AVgu2@a+rMzJaCK%)eRMTYEbCfgoivpcVQ3nu;rSb=M?t>T&)qOY>{>Ij=j#q zMSIoqN^}wqYw#-0UC-un#VeJpm6E-ONei6ACD8_wCP?;R)EN66oWUxalrn9+$uwPR zQKw*9loDsfqqb|Bq@BHxuE%0fK?I0faB~cYCY0?Q>Gf%r54FW_;!{F&5Hi8KvlL#r8Wzat_jTz#o^()rwaW#*%hmXVV>| z19s9W;JkpOfC~b|y%1GE3LzPhE)jOKm&%|F-q1p&dY^q$h7GP3ZMQ(28g;4_uBIbl4v6iKaiakQQr~h5V*T+EAmX)o;&!Vk6l%JaR(SN z&UV%3g=3u?+Oc|2$;0E>`SEOSLcE-6rD&JzW0B4ODe_q3Q*97y?2Xpn{jX^=Vp;*h zI=9TS;Vw27oon!~F~gojGdRoMMF)dtNs7t(2+QfIP2Wi05zVUA`r!(!6?nZQDw_H4 z!Z*S1Ju0}YfYUy4`C7SJ^h)+9nT4B5r)ZC*Rbq8|njzp7U=hTbSfObLPx{&VSB# zF8|HV=XX2XoDQpikK@j&W6es|8DMYUYVKqsUZGyls{MjiGt9cov_lXKg-dHu8Y+=i za%mk(>r13=E)AnJQX=(e4G|&Qczc9BEN7qIK4ANau^wo1TvE5-0XmT`Vw#9}xd^i5 zDmw^&IM%dl%&>B^CNyjAGU0!G4BI|o9n(a=Fkvz5nq)Zk35FB8o!x93oO14vcHpLM zkn*;e>KOcHUkfd+n~cHtuCQg09T0KRM@@yDPDS?|dO0$BH!QkR-kdvrB7SrtK01+% z8L8N~p31zIhQk%Z>!DFHDPaY?vjTxGZ{952KW@haEM zh86L$!Bk}z`x3I25Im?<;OELQu)7;Oj<$#G??^vuo1!+{wq`1D17JtfbGM2t%r+&D*XN^u!X%!zP27A zB_gA^(h8D)FJe0GEAHv7i&BZVN4U9uRbk_|V=7~8E5w7|=9fr+j0ky>lhR`;J-Li3 z7ag8R#bKtR4!#Q3$m1wVABIlJ4gUmRm$`a^Jh4uZ5hCMoy{d^#;d!kr1AkRzULZhAddfmQ#RhpIPNJetgqwMsh}!V<&J~F`$$>e8z|^~( z+?W33h~}0ajEpTsW-3Nz$)uBd>}Vn#HSzLqT3@x&+KS-L zpq;b_NijHQWwz_`v3W7DV1`*ogL)eO=tO8=Nit3^*Z2jj@otHq00Yj|fy;&&ds@^? z*yCoNB2V$>MsJ670j*r7(Z_VmQY(a9O-D@_f)E!Z!U>A-Y0Uf=uBOiw}zZa({w9_Zn4kw7V}COh9BH& zKv5BiM-#*CyCz=2VG&udf#PqFdFfoD4&8K)>=WT8!$p(R!r!tGicIgkk7QTLuymDl z-#NnZl^)bsF)XviVaYEShL?+J`En8Yur%{wDQTi1tog9M?gQ4ChHCEJ67I&o^kvs{c zQ<>?{Q*Y#D!RDOlTPTCC1C8*p((uf?xLqg0mFXrz5cR^t>J3Bu zW~Q|9QOpM;A9Q@i@WH{`jaMyC3O`1tBZ@6riCoUIc`}vAOeDK?z)yxN{Uf|tb0FYk zq1iVI0=^1iiLJfXR}k=3sDM9fq8kesi>pv!Vv*TsLBLnR%9uL4pX6%6VnJQpkFSDH MCOZ!VeDPEN2h5@n-v9sr delta 946 zcmY*YT}TvB6uw8@&7E~e$93FYopt?V-LHGlDiWh8UxZ#@dT_l0QP|qaDpjuP- zrNRe8*eah6siI8i86Kf$)@RTfJ*G4$>Ojx(h}mq@soTt)L1DV^soVk)+>yI%ofH(o zJB1rnK`mU9=-bZ%Ti~Z4*!R(f zONRzKuqE4<-$bz!A|XbE3A!*dMmsSo7ZzCV$T7Lg)-z29xs$NgT@R1RO#rvt3Zi?e zifq^|8@)z1pWef8kP@+Qrjg4NSFxv{)6Nx%cj0QmWo6g#zcD8pa~|6Pqk&N45NjDM zWhs$g_bAhpS4%`vK7XP;RZ^blPFWUt!+*wxxO2FvDiFnY>OA$ZuCT*4O8Oyc$hN(i zn9+tEo;v3M)59G5Pn6?JPZ5c>JxlgUUYX*$O2bSUqPXZi1H)9eY8b~I?-~pst5$qP z^qurwf*Tn1R#Z>3fMJ|QTp_R$|EjnXk$u~-OsBa87_ci4OU@MmKpV~%t0h)nqrU~)J^oXOFx=c(SXf?BXKz(0j`pR6WW6|h`c#~3D-Io-G#l%xfU0~+yl+zej# hx*+gqAuZ^YF3p14mCm%FS6asMs<5<2?sKgx{THiG)7Jn1 diff --git a/app/auth.py b/app/auth.py index b580143..b5e1510 100644 --- a/app/auth.py +++ b/app/auth.py @@ -20,7 +20,9 @@ TokenResponse, RefreshTokenRequest, UserResponse, + UserSearchResult, ) +import uuid logger = logging.getLogger(__name__) @@ -281,3 +283,38 @@ async def refresh_tokens( async def get_me(current_user: User = Depends(get_current_user)): """Возвращает данные текущего авторизованного пользователя.""" return current_user + + +@router.get("/users", response_model=list[UserSearchResult]) +async def search_users( + q: str | None = None, + group_id: str | None = None, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Поиск пользователей. Только для admin. + q — фильтр по username (ILIKE) + group_id — вернуть только членов определённой группы + """ + if current_user.role != "admin": + raise HTTPException(status_code=403, detail="Admin access required") + + from app.models import user_groups + + stmt = select(User) + if group_id: + try: + gid = uuid.UUID(group_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid group_id format") + stmt = ( + stmt + .join(user_groups, User.id == user_groups.c.user_id) + .where(user_groups.c.group_id == gid) + ) + if q: + stmt = stmt.where(User.username.ilike(f"%{q}%")) + + result = await db.execute(stmt.order_by(User.username).limit(50)) + return result.scalars().all() diff --git a/app/documents.py b/app/documents.py index 3da713e..3189145 100644 --- a/app/documents.py +++ b/app/documents.py @@ -1,13 +1,14 @@ from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, BackgroundTasks +from fastapi.responses import FileResponse from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from sqlalchemy import or_ +from sqlalchemy import or_, and_, cast, String +import mimetypes import uuid import os -import shutil from app.database import get_db -from app.models import User, Document, Chunk +from app.models import User, Document, Chunk, user_groups from app.schemas import ( DocumentResponse, DocumentUploadResponse, DocumentUpdateRequest, ChunkResponse, SearchRequest, SearchResultItem, @@ -24,6 +25,54 @@ os.makedirs(UPLOAD_DIR, exist_ok=True) +# ────────────────────────────────────────── +# Вспомогательная функция проверки доступа +# ────────────────────────────────────────── + +async def check_document_access(document: Document, current_user: User, db: AsyncSession) -> bool: + """ + Возвращает True если пользователь имеет доступ к документу. + Логика: + - Владелец документа → всегда есть доступ. + - Admin → всегда есть доступ. + - Оба списка пусты (is_available_to=None и available_to_groups=None) → публичный, доступ есть. + - ID пользователя в is_available_to → доступ есть. + - Пользователь состоит в группе из available_to_groups → доступ есть. + """ + if str(document.uploader_id) == str(current_user.id): + return True + if current_user.role == "admin": + return True + + both_empty = ( + (not document.is_available_to or document.is_available_to == 'null') + and (not document.available_to_groups or document.available_to_groups == 'null') + ) + if both_empty: + return True + + if document.is_available_to and str(current_user.id) in document.is_available_to: + return True + + if document.available_to_groups: + user_group_ids = await _get_user_group_ids(current_user.id, db) + if any(gid in document.available_to_groups for gid in user_group_ids): + return True + + return False + + +async def _get_user_group_ids(user_id: uuid.UUID, db: AsyncSession) -> list[str]: + result = await db.execute( + select(user_groups.c.group_id).where(user_groups.c.user_id == user_id) + ) + return [str(row[0]) for row in result.fetchall()] + + +# ────────────────────────────────────────── +# Upload +# ────────────────────────────────────────── + @router.post("/upload", response_model=DocumentUploadResponse) async def upload_document( background_tasks: BackgroundTasks, @@ -31,14 +80,15 @@ async def upload_document( title: str | None = Form(None), author: str | None = Form(None), description: str | None = Form(None), - is_available_to: str | None = Form(None), + is_available_to: str | None = Form(None), # строка UUID через запятую + available_to_groups: str | None = Form(None), # строка UUID через запятую current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db) ): document_id = uuid.uuid4() - - # Обработка is_available_to (парсинг из строки, разделенной запятыми) - available_users = [] + + # Парсинг is_available_to + available_users: list[str] = [] if is_available_to: for uid in is_available_to.split(","): uid = uid.strip() @@ -47,14 +97,29 @@ async def upload_document( uuid.UUID(uid) available_users.append(uid) except ValueError: - raise HTTPException(status_code=400, detail=f"Invalid UUID format: {uid}") - - # Определение расширения + raise HTTPException(status_code=400, detail=f"Invalid user UUID: {uid}") + + # Парсинг available_to_groups + # Обычный пользователь может назначать только группы, в которых сам состоит + available_groups: list[str] = [] + if available_to_groups: + raw_groups = [g.strip() for g in available_to_groups.split(",") if g.strip()] + if current_user.role != "admin": + user_group_ids = await _get_user_group_ids(current_user.id, db) + # Фильтруем — только свои группы + raw_groups = [g for g in raw_groups if g in user_group_ids] + for gid in raw_groups: + try: + uuid.UUID(gid) + available_groups.append(gid) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid group UUID: {gid}") + + # Расширение и сохранение файла filename = file.filename or "" _, ext = os.path.splitext(filename) extension = ext.lstrip(".").lower() if ext else None - # Сохранение файла на диск file_path = os.path.join(UPLOAD_DIR, f"{document_id}.{extension}" if extension else str(document_id)) try: with open(file_path, "wb") as f: @@ -64,47 +129,66 @@ async def upload_document( except Exception as e: raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}") - # Создание записи в БД - doc_title = title or filename + # Запись в БД document = Document( id=document_id, - title=doc_title, + title=title or filename, author=author, uploader_id=current_user.id, extension=extension, size_bytes=size_bytes, description=description, file_path=file_path, - is_available_to=available_users if available_users else None + is_available_to=available_users if available_users else None, + available_to_groups=available_groups if available_groups else None, ) - + db.add(document) await db.commit() - # Запуск фоновой задачи для парсинга и векторизации background_tasks.add_task(process_document, document_id=str(document_id), file_path=file_path) return DocumentUploadResponse(document_id=document_id, status="processing") +# ────────────────────────────────────────── +# List / Get +# ────────────────────────────────────────── + @router.get("", response_model=list[DocumentResponse]) async def list_documents( current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db) ): - query = select(Document) - if current_user.role != "admin": - # Обычный пользователь видит свои документы и те, что ему расшарены - query = query.where( - or_( - Document.uploader_id == current_user.id, - Document.is_available_to.has_key(str(current_user.id)) - ) + if current_user.role == "admin": + result = await db.execute(select(Document)) + return result.scalars().all() + + # Для обычных пользователей учитываем и группы + user_group_ids = await _get_user_group_ids(current_user.id, db) + + group_conditions = [ + Document.available_to_groups.has_key(gid) + for gid in user_group_ids + ] + + query = select(Document).where( + or_( + Document.uploader_id == current_user.id, + # Публичный — оба списка пусты + and_( + or_(Document.is_available_to.is_(None), cast(Document.is_available_to, String).in_(('null', '[]'))), + or_(Document.available_to_groups.is_(None), cast(Document.available_to_groups, String).in_(('null', '[]'))) + ), + # Явный доступ пользователю + Document.is_available_to.has_key(str(current_user.id)), + # Доступ через группу + *group_conditions, ) - + ) + result = await db.execute(query) - documents = result.scalars().all() - return documents + return result.scalars().all() @router.get("/{doc_id}", response_model=DocumentResponse) @@ -115,41 +199,19 @@ async def get_document( ): result = await db.execute(select(Document).where(Document.id == doc_id)) document = result.scalar_one_or_none() - if not document: raise HTTPException(status_code=404, detail="Document not found") - - if current_user.role != "admin" and document.uploader_id != current_user.id: - if not document.is_available_to or str(current_user.id) not in document.is_available_to: - raise HTTPException(status_code=403, detail="Access denied") - - return document + if not await check_document_access(document, current_user, db): + raise HTTPException(status_code=403, detail="Access denied") -@router.get("/{doc_id}/chunks", response_model=list[ChunkResponse]) -async def get_document_chunks( - doc_id: uuid.UUID, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db) -): - # Сначала проверяем доступ к самому документу - doc_result = await db.execute(select(Document).where(Document.id == doc_id)) - document = doc_result.scalar_one_or_none() - - if not document: - raise HTTPException(status_code=404, detail="Document not found") - - if current_user.role != "admin" and document.uploader_id != current_user.id: - if not document.is_available_to or str(current_user.id) not in document.is_available_to: - raise HTTPException(status_code=403, detail="Access denied") - - # Загружаем чанки - chunk_result = await db.execute( - select(Chunk).where(Chunk.document_id == doc_id).order_by(Chunk.chunk_index) - ) - return chunk_result.scalars().all() + return document +# ────────────────────────────────────────── +# Update +# ────────────────────────────────────────── + @router.put("/{doc_id}", response_model=DocumentResponse) async def update_document( doc_id: uuid.UUID, @@ -159,13 +221,13 @@ async def update_document( ): result = await db.execute(select(Document).where(Document.id == doc_id)) document = result.scalar_one_or_none() - + if not document: raise HTTPException(status_code=404, detail="Document not found") - + if current_user.role != "admin" and document.uploader_id != current_user.id: raise HTTPException(status_code=403, detail="Only the owner or an admin can update this document") - + if request.title is not None: document.title = request.title if request.author is not None: @@ -173,14 +235,25 @@ async def update_document( if request.description is not None: document.description = request.description if request.is_available_to is not None: - # Для обновления передаём список как есть. Если список пустой, сохраняем None. document.is_available_to = [str(u) for u in request.is_available_to] if request.is_available_to else None - + if request.available_to_groups is not None: + # Проверяем права: обычный пользователь — только свои группы + if current_user.role != "admin": + user_group_ids = await _get_user_group_ids(current_user.id, db) + filtered = [str(g) for g in request.available_to_groups if str(g) in user_group_ids] + else: + filtered = [str(g) for g in request.available_to_groups] + document.available_to_groups = filtered if filtered else None + await db.commit() await db.refresh(document) return document +# ────────────────────────────────────────── +# Delete +# ────────────────────────────────────────── + @router.delete("/{doc_id}") async def delete_document( doc_id: uuid.UUID, @@ -189,31 +262,30 @@ async def delete_document( ): result = await db.execute(select(Document).where(Document.id == doc_id)) document = result.scalar_one_or_none() - + if not document: raise HTTPException(status_code=404, detail="Document not found") - + if current_user.role != "admin" and document.uploader_id != current_user.id: raise HTTPException(status_code=403, detail="Only the owner or an admin can delete this document") - - # Удаление файла с диска + if document.file_path and os.path.exists(document.file_path): try: os.remove(document.file_path) except Exception as e: - # Логируем, но продолжаем удаление из БД print(f"Failed to delete file {document.file_path}: {e}") - - # Удаление векторов из Qdrant + delete_chunks_by_document(str(doc_id)) - - # Удаление из БД (связанные чанки удалятся каскадно) await db.delete(document) await db.commit() - + return {"status": "deleted"} +# ────────────────────────────────────────── +# Search +# ────────────────────────────────────────── + @router.post("/search", response_model=list[SearchResultItem]) async def search_documents( request: SearchRequest, @@ -233,3 +305,55 @@ async def search_documents( top_k=request.top_k, ) return results + + +# ────────────────────────────────────────── +# File Content (Preview / Download) +# ────────────────────────────────────────── + +INLINE_EXTENSIONS = {"pdf", "txt", "png", "jpg", "jpeg", "gif", "webp", "svg"} + + +@router.get("/{doc_id}/content") +async def get_document_content( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Возвращает оригинальный файл документа. + - PDF, TXT, изображения — отдаются inline (отображаются в браузере). + - DOCX, XLSX, PPTX и пр. — отдаются как вложение (скачивание). + Доступ проверяется по правам пользователя, включая группы. + """ + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if document is None: + raise HTTPException(status_code=404, detail="Document not found") + + if not await check_document_access(document, current_user, db): + raise HTTPException(status_code=403, detail="Access denied") + + if not document.file_path or not os.path.exists(document.file_path): + raise HTTPException(status_code=404, detail="File not found on disk") + + ext = (document.extension or "").lstrip(".").lower() + mime_type, _ = mimetypes.guess_type(document.file_path) + if not mime_type: + mime_type = "application/octet-stream" + + disposition = "inline" if ext in INLINE_EXTENSIONS else "attachment" + filename = os.path.basename(document.file_path) + + from urllib.parse import quote + + return FileResponse( + path=document.file_path, + media_type=mime_type, + filename=filename, + content_disposition_type=disposition, + headers={ + "X-Document-Title": quote(document.title or filename), + }, + ) diff --git a/app/groups.py b/app/groups.py new file mode 100644 index 0000000..56d567a --- /dev/null +++ b/app/groups.py @@ -0,0 +1,190 @@ +""" +app/groups.py — управление группами доступа. +Создание/удаление групп и управление участниками — только для admin. +Просмотр списка групп — для всех авторизованных пользователей. +""" +import uuid +import logging +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy import delete, func + +from app.database import get_db +from app.models import User, Group, user_groups +from app.schemas import GroupCreate, GroupResponse, GroupMemberAdd, UserSearchResult +from app.auth import get_current_user + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/groups", tags=["groups"]) + + +# ────────────────────────────────────────── +# Вспомогательные функции +# ────────────────────────────────────────── + +def require_admin(current_user: User = Depends(get_current_user)) -> User: + if current_user.role != "admin": + raise HTTPException(status_code=403, detail="Admin access required") + return current_user + + +async def get_user_group_ids(user_id: uuid.UUID, db: AsyncSession) -> list[str]: + """Возвращает список строковых ID групп, в которых состоит пользователь.""" + result = await db.execute( + select(user_groups.c.group_id).where(user_groups.c.user_id == user_id) + ) + return [str(row[0]) for row in result.fetchall()] + + +# ────────────────────────────────────────── +# Группы CRUD +# ────────────────────────────────────────── + +@router.get("", response_model=list[GroupResponse]) +async def list_groups( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Список всех групп. Доступен для всех авторизованных пользователей.""" + result = await db.execute(select(Group).order_by(Group.name)) + groups = result.scalars().all() + + # Подсчитываем количество участников для каждой группы + group_list = [] + for g in groups: + count_result = await db.execute( + select(func.count()).select_from(user_groups).where(user_groups.c.group_id == g.id) + ) + count = count_result.scalar() or 0 + group_list.append(GroupResponse( + id=g.id, + name=g.name, + description=g.description, + created_at=g.created_at, + member_count=count, + )) + return group_list + + +@router.post("", response_model=GroupResponse) +async def create_group( + data: GroupCreate, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Создать новую группу. Только для admin.""" + existing = await db.execute(select(Group).where(Group.name == data.name)) + if existing.scalar_one_or_none(): + raise HTTPException(status_code=400, detail="Группа с таким именем уже существует") + + group = Group( + id=uuid.uuid4(), + name=data.name, + description=data.description, + created_by=current_user.id, + ) + db.add(group) + await db.commit() + await db.refresh(group) + return GroupResponse( + id=group.id, + name=group.name, + description=group.description, + created_at=group.created_at, + member_count=0, + ) + + +@router.delete("/{group_id}") +async def delete_group( + group_id: uuid.UUID, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Удалить группу. Только для admin.""" + result = await db.execute(select(Group).where(Group.id == group_id)) + group = result.scalar_one_or_none() + if not group: + raise HTTPException(status_code=404, detail="Group not found") + await db.delete(group) + await db.commit() + return {"status": "deleted"} + + +# ────────────────────────────────────────── +# Участники групп +# ────────────────────────────────────────── + +@router.get("/{group_id}/members", response_model=list[UserSearchResult]) +async def get_group_members( + group_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Список участников группы.""" + group_result = await db.execute(select(Group).where(Group.id == group_id)) + if not group_result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="Group not found") + + stmt = ( + select(User) + .join(user_groups, User.id == user_groups.c.user_id) + .where(user_groups.c.group_id == group_id) + .order_by(User.username) + ) + result = await db.execute(stmt) + return result.scalars().all() + + +@router.post("/{group_id}/members") +async def add_members( + group_id: uuid.UUID, + data: GroupMemberAdd, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Добавить пользователей в группу. Только для admin.""" + group_result = await db.execute(select(Group).where(Group.id == group_id)) + if not group_result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="Group not found") + + added = [] + for user_id in data.user_ids: + user_result = await db.execute(select(User).where(User.id == user_id)) + if not user_result.scalar_one_or_none(): + continue + + existing = await db.execute( + select(user_groups).where( + user_groups.c.user_id == user_id, + user_groups.c.group_id == group_id, + ) + ) + if not existing.first(): + await db.execute( + user_groups.insert().values(user_id=user_id, group_id=group_id) + ) + added.append(str(user_id)) + + await db.commit() + return {"added": added} + + +@router.delete("/{group_id}/members/{user_id}") +async def remove_member( + group_id: uuid.UUID, + user_id: uuid.UUID, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Убрать пользователя из группы. Только для admin.""" + await db.execute( + delete(user_groups).where( + user_groups.c.user_id == user_id, + user_groups.c.group_id == group_id, + ) + ) + await db.commit() + return {"status": "removed"} diff --git a/app/main.py b/app/main.py index 637077e..a3e2ed8 100644 --- a/app/main.py +++ b/app/main.py @@ -72,10 +72,12 @@ async def lifespan(app: FastAPI): from app.auth import router as auth_router # noqa: E402 from app.documents import router as docs_router from app.chat import router as chat_router +from app.groups import router as groups_router app.include_router(auth_router) app.include_router(docs_router) app.include_router(chat_router) +app.include_router(groups_router) # ────────────────────────────────────────── diff --git a/app/models.py b/app/models.py index dc33e4d..3d58b8f 100644 --- a/app/models.py +++ b/app/models.py @@ -1,9 +1,23 @@ import uuid from datetime import datetime -from sqlalchemy import Column, String, Integer, DateTime, ForeignKey, BigInteger, Index -from sqlalchemy.dialects.postgresql import UUID, JSONB # <-- Импортируем JSONB +from sqlalchemy import Column, String, Integer, DateTime, ForeignKey, BigInteger, Index, Table +from sqlalchemy.dialects.postgresql import UUID, JSONB +from sqlalchemy.orm import relationship + from app.database import Base + +# ────────────────────────────────────────── +# Ассоциативная таблица User ↔ Group +# ────────────────────────────────────────── +user_groups = Table( + "user_groups", + Base.metadata, + Column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True), + Column("group_id", UUID(as_uuid=True), ForeignKey("groups.id", ondelete="CASCADE"), primary_key=True), +) + + class User(Base): __tablename__ = "users" id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) @@ -11,6 +25,20 @@ class User(Base): hashed_password = Column(String, nullable=False) role = Column(String, default="user") + groups = relationship("Group", secondary=user_groups, back_populates="members") + + +class Group(Base): + __tablename__ = "groups" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String, unique=True, nullable=False) + description = Column(String) + created_by = Column(UUID(as_uuid=True), ForeignKey("users.id")) + created_at = Column(DateTime, default=datetime.utcnow) + + members = relationship("User", secondary=user_groups, back_populates="groups") + + class Document(Base): __tablename__ = "documents" id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) @@ -23,7 +51,9 @@ class Document(Base): size_bytes = Column(BigInteger) description = Column(String) file_path = Column(String) - is_available_to = Column(JSONB) # список ID пользователей, имеющих доступ + is_available_to = Column(JSONB) # список ID пользователей, имеющих доступ + available_to_groups = Column(JSONB) # список ID групп, имеющих доступ + class Chunk(Base): __tablename__ = "chunks" @@ -31,16 +61,17 @@ class Chunk(Base): document_id = Column(UUID(as_uuid=True), ForeignKey("documents.id", ondelete="CASCADE")) chunk_index = Column(Integer) text = Column(String) - keywords = Column(JSONB) # список ключевых слов (строк) + keywords = Column(JSONB) language = Column(String) start_char = Column(Integer) end_char = Column(Integer) + # Индексы для производительности Index("idx_chunks_document_id", Chunk.document_id) Index( "idx_chunks_keywords_gin", Chunk.keywords, postgresql_using="gin", - postgresql_ops={"keywords": "jsonb_ops"} + postgresql_ops={"keywords": "jsonb_ops"}, ) \ No newline at end of file diff --git a/app/schemas.py b/app/schemas.py index 0f37415..851deda 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -64,7 +64,8 @@ class DocumentResponse(BaseModel): extension: str | None = None size_bytes: int | None = None description: str | None = None - is_available_to: list[UUID] | None = None + is_available_to: list[str] | None = None + available_to_groups: list[str] | None = None class Config: from_attributes = True @@ -75,6 +76,7 @@ class DocumentUpdateRequest(BaseModel): author: str | None = None description: str | None = None is_available_to: list[UUID] | None = None + available_to_groups: list[UUID] | None = None class ChunkResponse(BaseModel): @@ -114,3 +116,36 @@ class SearchResultItem(BaseModel): class ChatRequest(BaseModel): query: str + + +# ────────────────────────────────────────── +# Группы и управление доступом +# ────────────────────────────────────────── + +class GroupCreate(BaseModel): + name: str + description: str | None = None + + +class GroupMemberAdd(BaseModel): + user_ids: list[UUID] + + +class GroupResponse(BaseModel): + id: UUID + name: str + description: str | None = None + created_at: datetime + member_count: int | None = None + + class Config: + from_attributes = True + + +class UserSearchResult(BaseModel): + id: UUID + username: str + role: str + + class Config: + from_attributes = True diff --git a/app/search.py b/app/search.py index ac51384..4fcdc4f 100644 --- a/app/search.py +++ b/app/search.py @@ -5,12 +5,13 @@ from typing import List from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from sqlalchemy import or_, cast, String +from sqlalchemy import or_, and_, cast, String -from app.models import User, Document, Chunk +from app.models import User, Document, Chunk, user_groups from app.qdrant_client import semantic_search from app.embeddings import get_embedding from app.schemas import SearchResultItem +from sqlalchemy.future import select logger = logging.getLogger(__name__) @@ -34,15 +35,33 @@ async def hybrid_search( """ # ── 1. Список доступных документов ─────────────────────────────────── - doc_query = select(Document) - if current_user.role != "admin": - doc_query = doc_query.where( + if current_user.role == "admin": + doc_result = await db.execute(select(Document)) + else: + # Получаем группы пользователя + gid_result = await db.execute( + select(user_groups.c.group_id).where(user_groups.c.user_id == current_user.id) + ) + user_group_ids = [str(row[0]) for row in gid_result.fetchall()] + + group_conditions = [ + Document.available_to_groups.has_key(gid) + for gid in user_group_ids + ] + + doc_query = select(Document).where( or_( Document.uploader_id == current_user.id, - Document.is_available_to.has_key(str(current_user.id)) + and_( + or_(Document.is_available_to.is_(None), cast(Document.is_available_to, String).in_(('null', '[]'))), + or_(Document.available_to_groups.is_(None), cast(Document.available_to_groups, String).in_(('null', '[]'))) + ), + Document.is_available_to.has_key(str(current_user.id)), + *group_conditions, ) ) - doc_result = await db.execute(doc_query) + doc_result = await db.execute(doc_query) + accessible_docs = doc_result.scalars().all() accessible_doc_ids = _get_accessible_doc_ids(accessible_docs) doc_lookup = {str(doc.id): doc for doc in accessible_docs} diff --git a/front-end/app.js b/front-end/app.js index b63a778..34ddb64 100644 --- a/front-end/app.js +++ b/front-end/app.js @@ -6,6 +6,8 @@ let state = { isLoginMode: true, user: null, documents: [], + groups: [], + users: [], activeTab: 'dashboard', }; @@ -15,6 +17,7 @@ const views = { dashboard: document.getElementById('dashboard-view'), search: document.getElementById('search-view'), chat: document.getElementById('chat-view'), + groups: document.getElementById('groups-view'), }; const navActions = document.getElementById('nav-actions'); const tabNav = document.getElementById('tab-nav'); @@ -138,6 +141,8 @@ function switchTab(tabName) { if (tabName === 'dashboard') { fetchDocuments(); + } else if (tabName === 'groups' && state.user && state.user.role === 'admin') { + fetchGroups(); } } @@ -151,22 +156,6 @@ async function fetchDocuments() { } catch (e) { console.error(e); } } -async function uploadFile(file) { - const formData = new FormData(); - formData.append('file', file); - - showToast('Uploading...', 'success'); - try { - await apiFetch('/documents/upload', { - method: 'POST', - body: formData, - }); - showToast('File uploaded successfully!'); - fetchDocuments(); - } catch (e) { - console.error(e); - } -} async function deleteDocument(id) { if (!confirm('Are you sure you want to delete this document?')) return; @@ -179,6 +168,104 @@ async function deleteDocument(id) { window.deleteDocument = deleteDocument; +// ───────────────────────────────────────────────────────────── +// Document Preview Modal +// ───────────────────────────────────────────────────────────── + +const INLINE_EXTS = new Set(['pdf', 'txt', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg']); + +const EXT_ICONS = { + pdf: '📕', txt: '📝', png: '🖼', jpg: '🖼', jpeg: '🖼', gif: '🖼', + webp: '🖼', svg: '🖼', docx: '📘', doc: '📘', xlsx: '📗', xls: '📗', + pptx: '📙', ppt: '📙', +}; + +async function openDocModal(documentId, documentTitle, extension) { + const modal = document.getElementById('doc-modal'); + const icon = document.getElementById('modal-icon'); + const title = document.getElementById('modal-title'); + const body = document.getElementById('modal-body'); + const downloadBtn = document.getElementById('modal-download-btn'); + + const ext = (extension || '').replace('.', '').toLowerCase(); + + // Показываем модалку сразу с лоадером + icon.textContent = EXT_ICONS[ext] || '📄'; + title.textContent = documentTitle || 'Document'; + body.innerHTML = `
+ Загрузка... +
`; + downloadBtn.href = '#'; + downloadBtn.removeAttribute('download'); + modal.style.display = 'flex'; + document.body.style.overflow = 'hidden'; + + // Загружаем файл через fetch с авторизацией + try { + const token = localStorage.getItem('access_token'); + const response = await fetch(`${API_URL}/documents/${documentId}/content`, { + headers: { 'Authorization': `Bearer ${token}` } + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + + // Ссылка на скачивание + downloadBtn.href = blobUrl; + downloadBtn.download = documentTitle || `document.${ext}`; + + // Тело модалки + if (INLINE_EXTS.has(ext)) { + body.innerHTML = ``; + // Освобождаем blob URL когда iframe загрузился + body.querySelector('iframe').onload = () => {}; + } else { + body.innerHTML = ` +
+
${EXT_ICONS[ext] || '📄'}
+

${escapeHtml(documentTitle || 'Document')}

+

Формат .${ext.toUpperCase()} нельзя отобразить прямо в браузере. + Нажмите кнопку ниже, чтобы скачать файл.

+ + ⬇ Скачать файл + +
`; + } + + // Сохраняем blobUrl для освобождения при закрытии + modal.dataset.blobUrl = blobUrl; + + } catch (e) { + body.innerHTML = `
+ ❌ Ошибка загрузки файла: ${escapeHtml(e.message)} +
`; + } +} + +function closeDocModal() { + const modal = document.getElementById('doc-modal'); + // Освобождаем Blob URL чтобы не было утечки памяти + if (modal.dataset.blobUrl) { + URL.revokeObjectURL(modal.dataset.blobUrl); + delete modal.dataset.blobUrl; + } + modal.style.display = 'none'; + document.getElementById('modal-body').innerHTML = ''; + document.body.style.overflow = ''; +} +window.openDocModal = openDocModal; +window.closeDocModal = closeDocModal; + +// Закрытие по Esc +document.addEventListener('keydown', (e) => { + if (e.key === 'Escape') closeDocModal(); +}); + // ───────────────────────────────────────────────────────────── // Search Functions // ───────────────────────────────────────────────────────────── @@ -221,17 +308,26 @@ function renderSearchResults(results, query) { stateEl.style.display = 'none'; container.innerHTML = results.map((r, i) => { - const ext = r.extension ? r.extension.toUpperCase() : '?'; + const extRaw = (r.extension || '').replace('.', '').toLowerCase(); + const extLabel = extRaw ? extRaw.toUpperCase() : '?'; const scorePercent = Math.min(100, Math.round(r.score * 100)); const keywords = (r.keywords || []).slice(0, 8); const snippet = r.text.length > 400 ? r.text.slice(0, 400) + '…' : r.text; + const docIcon = EXT_ICONS[extRaw] || '📄'; return `
- ${ext} - ${escapeHtml(r.document_title)} + ${extLabel} + + ${docIcon} ${escapeHtml(r.document_title)} +
⚡ ${scorePercent}% match @@ -347,9 +443,20 @@ async function performChat(query) { if (data.type === 'sources') { if (data.sources && data.sources.length > 0) { + const sourceLinks = data.sources.map(s => { + const ext = (s.extension || '').replace('.', '').toLowerCase(); + return ` + ${EXT_ICONS[ext] || '📄'} ${escapeHtml(s.document_title)} + `; + }).join(' '); sourcesHtml = `
📚 Источники:
- ${data.sources.map(s => `📄 ${escapeHtml(s.document_title)}`).join(' ')} + ${sourceLinks}
`; assistantMsgContent.innerHTML = sourcesHtml + fullText; } @@ -460,31 +567,6 @@ document.getElementById('search-form').addEventListener('submit', (e) => { }); // Drag & Drop -const dropArea = document.getElementById('uploadArea'); -const fileInput = document.getElementById('fileInput'); - -['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => { - dropArea.addEventListener(eventName, preventDefaults, false); -}); - -function preventDefaults(e) { e.preventDefault(); e.stopPropagation(); } - -['dragenter', 'dragover'].forEach(eventName => { - dropArea.addEventListener(eventName, () => dropArea.classList.add('drag-over'), false); -}); - -['dragleave', 'drop'].forEach(eventName => { - dropArea.addEventListener(eventName, () => dropArea.classList.remove('drag-over'), false); -}); - -dropArea.addEventListener('drop', (e) => { - const files = e.dataTransfer.files; - if (files.length) uploadFile(files[0]); -}); - -fileInput.addEventListener('change', (e) => { - if (e.target.files.length) uploadFile(e.target.files[0]); -}); // ───────────────────────────────────────────────────────────── // Render @@ -504,23 +586,34 @@ function renderDocuments() { return; } - grid.innerHTML = state.documents.map(doc => ` + grid.innerHTML = state.documents.map(doc => { + const extRaw = (doc.extension || '').replace('.', '').toLowerCase(); + const extLabel = extRaw ? extRaw.toUpperCase() : 'UNKNOWN'; + const docIcon = EXT_ICONS[extRaw] || '📄'; + return `
-
${escapeHtml(doc.title || 'Untitled')}
- ${doc.extension ? doc.extension.toUpperCase() : 'UNKNOWN'} +
${docIcon} ${escapeHtml(doc.title || 'Untitled')}
+ ${extLabel}
Size: ${formatBytes(doc.size_bytes)} Date: ${new Date(doc.upload_date).toLocaleDateString()}
+ ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) ? `` : ''}
-
- `).join(''); +
`; + }).join(''); } function render() { @@ -536,6 +629,11 @@ function render() { `; + // Toggle admin tabs + document.querySelectorAll('.admin-only').forEach(el => { + el.style.display = state.user.role === 'admin' ? 'inline-block' : 'none'; + }); + switchTab(state.activeTab); } else { Object.values(views).forEach(v => v.classList.remove('active')); @@ -545,5 +643,441 @@ function render() { } } + +// ───────────────────────────────────────────────────────────── +// Upload Modal & Logic +// ───────────────────────────────────────────────────────────── + +let pendingUploadFile = null; +let selectedUploadGroups = new Set(); +let selectedUploadUsers = new Set(); + +const openUploadBtn = document.getElementById('open-upload-modal-btn'); +if(openUploadBtn) { + openUploadBtn.addEventListener('click', () => openUploadModal()); +} + +async function openUploadModal(file = null) { + pendingUploadFile = file; + selectedUploadGroups.clear(); + selectedUploadUsers.clear(); + + const modal = document.getElementById('upload-modal'); + + // Fetch users/groups for dropdowns + if (state.user) { + if (!state.groups.length) await fetchGroups(false); // don't render view, just fetch + } + + modal.innerHTML = ` +
+
+

Upload Document

+ +
+
+ + + + + +
+ + +
+
+ + +
+ + +
+ +
+ + +
+ +
+
Select groups or specific users who can access this document.
+ + +
+ ${state.groups.map(g => ` + + `).join('')} + ${state.groups.length === 0 ? '
No groups available
' : ''} +
+ + +
+ + +
+
+
+
+ +
+
+ + +
+
+ `; + + modal.style.display = 'flex'; + document.body.style.overflow = 'hidden'; + + // File input handlers + const dropZone = document.getElementById('modal-drop-zone'); + const fileIn = document.getElementById('modalFileInput'); + + dropZone.addEventListener('dragover', (e) => { e.preventDefault(); dropZone.classList.add('dragover'); }); + dropZone.addEventListener('dragleave', () => dropZone.classList.remove('dragover')); + dropZone.addEventListener('drop', (e) => { + e.preventDefault(); + dropZone.classList.remove('dragover'); + if (e.dataTransfer.files.length) { + pendingUploadFile = e.dataTransfer.files[0]; + document.getElementById('modal-drop-text').textContent = 'File selected'; + document.getElementById('modal-file-name').textContent = pendingUploadFile.name; + } + }); + fileIn.addEventListener('change', (e) => { + if (e.target.files.length) { + pendingUploadFile = e.target.files[0]; + document.getElementById('modal-drop-text').textContent = 'File selected'; + document.getElementById('modal-file-name').textContent = pendingUploadFile.name; + } + }); +} + +function closeUploadModal() { + document.getElementById('upload-modal').style.display = 'none'; + document.body.style.overflow = ''; + pendingUploadFile = null; +} + +window.toggleAccessType = function(type) { + const btnAll = document.getElementById('btn-access-all'); + const btnRestricted = document.getElementById('btn-access-restricted'); + const panel = document.getElementById('restricted-panel'); + + if (type === 'all') { + btnAll.classList.add('active'); + btnRestricted.classList.remove('active'); + panel.classList.remove('visible'); + selectedUploadGroups.clear(); + selectedUploadUsers.clear(); + // Uncheck all group boxes + document.querySelectorAll('#upload-group-list input[type="checkbox"]').forEach(cb => cb.checked = false); + renderUploadSelectedUsers(); + } else { + btnRestricted.classList.add('active'); + btnAll.classList.remove('active'); + panel.classList.add('visible'); + } +} + +window.toggleUploadGroup = function(groupId, isChecked) { + if (isChecked) selectedUploadGroups.add(groupId); + else selectedUploadGroups.delete(groupId); +} + +// User Search +let userSearchTimeout = null; +window.debounceUserSearch = function(query, context) { + clearTimeout(userSearchTimeout); + userSearchTimeout = setTimeout(() => searchUsersApi(query, context), 300); +} + +async function searchUsersApi(query, context) { + const dropdown = document.getElementById(context === 'upload' ? 'upload-user-results' : 'group-user-results'); + if (!query || query.length < 2) { + dropdown.style.display = 'none'; + return; + } + + try { + const users = await apiFetch(`/auth/users?q=${encodeURIComponent(query)}`); + if (users.length === 0) { + dropdown.innerHTML = '
No users found
'; + } else { + dropdown.innerHTML = users.map(u => ` +
+ ${escapeHtml(u.username)} (${u.role}) +
+ `).join(''); + } + dropdown.style.display = 'block'; + } catch (e) { + console.error("User search failed", e); + } +} + +window.selectUser = function(id, username, context) { + document.getElementById(context === 'upload' ? 'upload-user-results' : 'group-user-results').style.display = 'none'; + document.getElementById(context === 'upload' ? 'upload-user-search' : 'group-user-search').value = ''; + + if (context === 'upload') { + selectedUploadUsers.add({id, username}); + renderUploadSelectedUsers(); + } else { + selectedGroupUsers.add({id, username}); + renderGroupSelectedUsers(); + } +} + +window.removeUploadUser = function(id) { + for (let u of selectedUploadUsers) { + if (u.id === id) { selectedUploadUsers.delete(u); break; } + } + renderUploadSelectedUsers(); +} + +function renderUploadSelectedUsers() { + const container = document.getElementById('upload-selected-users'); + if(!container) return; + container.innerHTML = Array.from(selectedUploadUsers).map(u => ` +
+ 👤 ${escapeHtml(u.username)} + ✕ +
+ `).join(''); +} + +async function submitUpload() { + if (!pendingUploadFile) { + showToast('Please select a file first', 'error'); + return; + } + + const title = document.getElementById('upload-title').value.trim(); + const desc = document.getElementById('upload-desc').value.trim(); + + const formData = new FormData(); + formData.append('file', pendingUploadFile); + if (title) formData.append('title', title); + if (desc) formData.append('description', desc); + + const isRestricted = document.getElementById('btn-access-restricted').classList.contains('active'); + if (isRestricted) { + if (selectedUploadUsers.size > 0) { + formData.append('is_available_to', Array.from(selectedUploadUsers).map(u => u.id).join(',')); + } + if (selectedUploadGroups.size > 0) { + formData.append('available_to_groups', Array.from(selectedUploadGroups).join(',')); + } + } + + showToast('Uploading...', 'success'); + closeUploadModal(); + + try { + await apiFetch('/documents/upload', { + method: 'POST', + body: formData, + }); + showToast('File uploaded successfully!'); + if (state.activeTab === 'dashboard') fetchDocuments(); + } catch (e) { + console.error(e); + } +} + + +// ───────────────────────────────────────────────────────────── +// Groups Management +// ───────────────────────────────────────────────────────────── + +async function fetchGroups(render = true) { + try { + state.groups = await apiFetch('/groups'); + if (render) renderGroups(); + } catch (e) { console.error(e); } +} + +function renderGroups() { + const container = document.getElementById('groups-list'); + if (!container) return; + + if (state.groups.length === 0) { + container.innerHTML = '

No groups found.

'; + return; + } + + container.innerHTML = state.groups.map(g => ` +
+
+
+
👥 ${escapeHtml(g.name)}
+
${escapeHtml(g.description || 'No description')} • ${g.member_count} members
+
+
+ +
+
+
+ Loading members... +
+
+ `).join(''); +} + +window.deleteGroup = async function(id) { + if (!confirm('Delete this group?')) return; + try { + await apiFetch(`/groups/${id}`, { method: 'DELETE' }); + showToast('Group deleted'); + fetchGroups(); + } catch (e) { console.error(e); } +} + +window.toggleGroupMembers = async function(groupId) { + const panel = document.getElementById(`group-panel-${groupId}`); + if (panel.classList.contains('open')) { + panel.classList.remove('open'); + return; + } + + // Close others + document.querySelectorAll('.group-members-panel').forEach(p => p.classList.remove('open')); + panel.classList.add('open'); + + try { + const members = await apiFetch(`/groups/${groupId}/members`); + panel.innerHTML = ` +
+ ${members.length === 0 ? '
No members yet
' : ''} + ${members.map(m => ` +
+
+ 👤 ${escapeHtml(m.username)} + ${m.role} +
+ +
+ `).join('')} +
+
+
+ + +
+
+ `; + } catch (e) { + panel.innerHTML = '
Failed to load members
'; + } +} + +let addMemberTimeout = null; +window.debounceAddMemberSearch = function(query, groupId) { + clearTimeout(addMemberTimeout); + addMemberTimeout = setTimeout(() => searchAddMemberApi(query, groupId), 300); +} + +async function searchAddMemberApi(query, groupId) { + const dropdown = document.getElementById(`results-add-${groupId}`); + if (!query || query.length < 2) { + dropdown.style.display = 'none'; + return; + } + try { + const users = await apiFetch(`/auth/users?q=${encodeURIComponent(query)}`); + if (users.length === 0) { + dropdown.innerHTML = '
No users found
'; + } else { + dropdown.innerHTML = users.map(u => ` +
+ ${escapeHtml(u.username)} (${u.role}) +
+ `).join(''); + } + dropdown.style.display = 'block'; + } catch(e) {} +} + +window.addMemberToGroup = async function(groupId, userId) { + try { + await apiFetch(`/groups/${groupId}/members`, { + method: 'POST', + body: JSON.stringify({ user_ids: [userId] }) + }); + showToast('Member added'); + toggleGroupMembers(groupId); // Refresh open panel + toggleGroupMembers(groupId); + fetchGroups(true); // update member counts + } catch (e) { console.error(e); } +} + +window.removeMember = async function(groupId, userId) { + try { + await apiFetch(`/groups/${groupId}/members/${userId}`, { method: 'DELETE' }); + showToast('Member removed'); + toggleGroupMembers(groupId); // Refresh open panel + toggleGroupMembers(groupId); + fetchGroups(true); + } catch (e) { console.error(e); } +} + +// Create Group logic +document.getElementById('create-group-btn')?.addEventListener('click', () => { + const modal = document.getElementById('group-modal'); + modal.innerHTML = ` +
+
+

Create Group

+ +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+ `; + modal.style.display = 'flex'; +}); + +window.submitCreateGroup = async function() { + const name = document.getElementById('new-group-name').value.trim(); + const desc = document.getElementById('new-group-desc').value.trim(); + if (!name) return showToast('Name is required', 'error'); + + try { + await apiFetch('/groups', { + method: 'POST', + body: JSON.stringify({ name, description: desc }) + }); + showToast('Group created'); + document.getElementById('group-modal').style.display = 'none'; + fetchGroups(); + } catch (e) { console.error(e); } +} + + // Init checkAuth(); diff --git a/front-end/index.html b/front-end/index.html index 8c9ac94..5313fe0 100644 --- a/front-end/index.html +++ b/front-end/index.html @@ -5,7 +5,7 @@ Enterprise Docs Search - + @@ -15,6 +15,7 @@ + diff --git a/front-end/style.css b/front-end/style.css index 2bcddc9..e31ea51 100644 --- a/front-end/style.css +++ b/front-end/style.css @@ -223,6 +223,53 @@ header { } /* Documents Grid */ +/* Documents filter bar */ +.docs-search-bar { + margin-bottom: 1.25rem; +} + +/* Pagination */ +.docs-pagination { + display: flex; + justify-content: center; + align-items: center; + gap: 0.35rem; + margin-top: 2rem; + flex-wrap: wrap; +} + +.pagination-btn { + min-width: 2.25rem; + height: 2.25rem; + padding: 0 0.6rem; + border-radius: 8px; + border: 1px solid var(--glass-border); + background: rgba(255,255,255,0.05); + color: var(--text-primary); + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: all 0.15s ease; +} + +.pagination-btn:hover:not(:disabled) { + background: var(--primary); + border-color: var(--primary); + color: white; +} + +.pagination-btn.active { + background: var(--primary); + border-color: var(--primary); + color: white; + font-weight: 600; +} + +.pagination-btn:disabled { + opacity: 0.35; + cursor: not-allowed; +} + .docs-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); From 5ccefe276b0f63f91bdf4b5bceb5556a22667550 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sun, 14 Jun 2026 20:48:40 +0400 Subject: [PATCH 28/34] Add: group access editing --- .gitignore | 2 +- app/__pycache__/documents.cpython-314.pyc | Bin 23218 -> 23291 bytes app/documents.py | 2 +- front-end/app.js | 128 +++++++++++++++++++++- 4 files changed, 125 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4654bf2..f7bda04 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,6 @@ share/ .env* pyvenv.cfg tree.txt -__pycache__ +*__pycache__* uploads/* report/ \ No newline at end of file diff --git a/app/__pycache__/documents.cpython-314.pyc b/app/__pycache__/documents.cpython-314.pyc index ef6509bb4f14db648d43d0f82a97919e94350f8d..433bec4ff4aba752073a7b07c7929dfef4567027 100644 GIT binary patch delta 390 zcmdnAmGSphMqX_`UM>b8xPMbGYu85JkCu9-9~qPwP4Dms^m}%CeqaW&pYSSuU`2nxlMi+&j6}xa%A!pXFSNl=wZupkX0JQ zR&)jl9Z}%}kwzk(3@k_4z^aaMFnY4G9OD!MN*t3G@?>W@X2cI9k24AaMUD%Jc=E6u zSK|b+4P=1i2_8mIL6#G|5+JrJqo**-2{n-36BbI0o}w%#ELnk^lMHMilARkw3NwO8 fQ4l%V-ZX3T7AGHZZqv`q3`|xZm_R|t9km<))H7qf delta 343 zcmeypm2uNnMqX_`UM>b8aJi&Fdk%K^w42B$SMtDD>{ROj;Qc~NFxyseU_taU{%LB7(Gl_ zj&TYBC5}l8d6=;rGvWu5#~B5IBFBY9Jgixct8s$Z1~NeM1P`Mp1Ir0s2@qSA(UXbg xgc>`DtH`xfEJC7|0{}k7R$Bl7 diff --git a/app/documents.py b/app/documents.py index 603dd2b..f0bbe0c 100644 --- a/app/documents.py +++ b/app/documents.py @@ -291,7 +291,7 @@ async def update_document( await db.commit() await db.refresh(document) - return document + return await _enrich_with_uploader(document, db) # ────────────────────────────────────────── diff --git a/front-end/app.js b/front-end/app.js index ec63a43..4111638 100644 --- a/front-end/app.js +++ b/front-end/app.js @@ -310,7 +310,14 @@ window.closeDocModal = closeDocModal; // Закрытие по Esc document.addEventListener('keydown', (e) => { - if (e.key === 'Escape') closeDocModal(); + if (e.key === 'Escape') { + closeDocModal(); + if (document.getElementById('upload-modal').style.display !== 'none') closeUploadModal(); + if (document.getElementById('group-modal').style.display !== 'none') { + // Could be Create Group or Doc Access modal — both have close buttons + if (typeof closeDocAccessModal === 'function') closeDocAccessModal(); + } + } }); // ───────────────────────────────────────────────────────────── @@ -669,7 +676,7 @@ function renderDocuments() { Date: ${new Date(doc.upload_date).toLocaleDateString()} ${uploaderLabel ? `${uploaderLabel}` : ''}
-
+
- ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) - ? `` - : ''} + ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) ? ` + + + ` : ''}
`; }).join(''); @@ -711,6 +722,113 @@ window.goToDocsPage = function(page) { document.getElementById('docs-grid').scrollIntoView({ behavior: 'smooth', block: 'start' }); } +// ───────────────────────────────────────────────────────────── +// Document Access (Groups) Modal +// ───────────────────────────────────────────────────────────── + +let docAccessDocId = null; +let docAccessSelectedGroups = new Set(); + +window.openDocAccessModal = async function(docId, currentGroupsRaw) { + docAccessDocId = docId; + docAccessSelectedGroups = new Set( + Array.isArray(currentGroupsRaw) ? currentGroupsRaw : [] + ); + + // Получаем актуальные группы пользователя (уже в state.groups после fetchGroups) + if (!state.groups.length) await fetchGroups(false); + + const modal = document.getElementById('group-modal'); + + // Все группы, которые пользователь может назначить + // Админ видит все (но пока используем только те, что в state.groups) + const availableGroups = state.groups; + + modal.innerHTML = ` +
+
+

⚙️ Document Access

+ +
+
+

+ Выберите группы, которые имеют доступ к документу.
+ Если не выбрать ни одной — документ станет публичным. +

+ + + ${availableGroups.length === 0 ? ` +
+ Вы не состоите ни в одной группе. Документ останется публичным. +
+ ` : ` +
+ ${availableGroups.map(g => ` + + `).join('')} +
+ `} + +
+ ${docAccessSelectedGroups.size === 0 ? '🌐 Документ публичный' : `🔒 Доступ ограничен: ${docAccessSelectedGroups.size} гр.`} +
+
+
+ + +
+
+ `; + + modal.style.display = 'flex'; + document.body.style.overflow = 'hidden'; +} + +window.toggleDocAccessGroup = function(groupId, isChecked) { + if (isChecked) docAccessSelectedGroups.add(groupId); + else docAccessSelectedGroups.delete(groupId); + + const statusEl = document.getElementById('doc-access-status'); + if (statusEl) { + statusEl.textContent = docAccessSelectedGroups.size === 0 + ? '🌐 Документ публичный' + : `🔒 Доступ ограничен: ${docAccessSelectedGroups.size} гр.`; + } +} + +window.closeDocAccessModal = function() { + document.getElementById('group-modal').style.display = 'none'; + document.body.style.overflow = ''; + docAccessDocId = null; + docAccessSelectedGroups.clear(); +} + +window.submitDocAccess = async function() { + if (!docAccessDocId) return; + try { + await apiFetch(`/documents/${docAccessDocId}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + available_to_groups: Array.from(docAccessSelectedGroups), + }), + }); + showToast('Доступ обновлён'); + closeDocAccessModal(); + fetchDocuments(); + } catch (e) { + showToast('Ошибка при обновлении доступа', 'error'); + console.error(e); + } +} + function render() { if (state.isAuthenticated) { views.auth.classList.remove('active'); From f0d3da32465f8b343f16b017e6430afbe0e8c308 Mon Sep 17 00:00:00 2001 From: Nomen Conservandum Date: Sun, 14 Jun 2026 20:55:49 +0400 Subject: [PATCH 29/34] Remove alien front-end elements --- front-end/app.js | 46 ++++++++++++++++++++++---------------------- front-end/index.html | 16 +++++++-------- front-end/style.css | 8 +++----- 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/front-end/app.js b/front-end/app.js index 4111638..eab3839 100644 --- a/front-end/app.js +++ b/front-end/app.js @@ -202,9 +202,9 @@ window.deleteDocument = deleteDocument; const INLINE_EXTS = new Set(['pdf', 'txt', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg']); const EXT_ICONS = { - pdf: '📕', txt: '📝', png: '🖼', jpg: '🖼', jpeg: '🖼', gif: '🖼', - webp: '🖼', svg: '🖼', docx: '📘', doc: '📘', xlsx: '📗', xls: '📗', - pptx: '📙', ppt: '📙', + pdf: 'PDF', txt: 'TXT', png: 'IMG', jpg: 'IMG', jpeg: 'IMG', gif: 'IMG', + webp: 'IMG', svg: 'IMG', docx: 'DOC', doc: 'DOC', xlsx: 'XLS', xls: 'XLS', + pptx: 'PPT', ppt: 'PPT', }; async function openDocModal(documentId, documentTitle, extension) { @@ -239,11 +239,11 @@ async function openDocModal(documentId, documentTitle, extension) { const metaHtml = docMeta ? `
- ${docMeta.author ? `✍️ Автор: ${escapeHtml(docMeta.author)}` : ''} - ${docMeta.uploader_username ? `👤 Загрузил: ${escapeHtml(docMeta.uploader_username)}` : ''} - ${docMeta.upload_date ? `📅 Дата: ${new Date(docMeta.upload_date).toLocaleDateString('ru-RU')}` : ''} - ${docMeta.size_bytes ? `📦 Размер: ${formatBytes(docMeta.size_bytes)}` : ''} - ${docMeta.description ? `📝 ${escapeHtml(docMeta.description)}` : ''} + ${docMeta.author ? `Автор: ${escapeHtml(docMeta.author)}` : ''} + ${docMeta.uploader_username ? `Загрузил: ${escapeHtml(docMeta.uploader_username)}` : ''} + ${docMeta.upload_date ? `Дата: ${new Date(docMeta.upload_date).toLocaleDateString('ru-RU')}` : ''} + ${docMeta.size_bytes ? `Размер: ${formatBytes(docMeta.size_bytes)}` : ''} + ${docMeta.description ? `${escapeHtml(docMeta.description)}` : ''}
` : ''; // Загружаем файл через fetch с авторизацией @@ -272,7 +272,7 @@ async function openDocModal(documentId, documentTitle, extension) { } else { body.innerHTML = `
-
${EXT_ICONS[ext] || '📄'}
+
${EXT_ICONS[ext] || 'DOC'}

${escapeHtml(documentTitle || 'Document')}

${metaHtml}

Формат .${ext.toUpperCase()} нельзя отобразить прямо в браузере. @@ -289,7 +289,7 @@ async function openDocModal(documentId, documentTitle, extension) { } catch (e) { body.innerHTML = `

- ❌ Ошибка загрузки файла: ${escapeHtml(e.message)} + Ошибка загрузки файла: ${escapeHtml(e.message)}
`; } } @@ -367,7 +367,7 @@ function renderSearchResults(results, query) { const scorePercent = Math.min(100, Math.round(r.score * 100)); const keywords = (r.keywords || []).slice(0, 8); const snippet = r.text.length > 400 ? r.text.slice(0, 400) + '…' : r.text; - const docIcon = EXT_ICONS[extRaw] || '📄'; + const docIcon = EXT_ICONS[extRaw] || ''; return `
@@ -505,7 +505,7 @@ async function performChat(query) { data-doc-ext="${ext}" onclick="openDocModal(this.dataset.docId, this.dataset.docTitle, this.dataset.docExt)" title="Открыть документ"> - ${EXT_ICONS[ext] || '📄'} ${escapeHtml(s.document_title)} + ${EXT_ICONS[ext] || ''} ${escapeHtml(s.document_title)} `; }).join(' '); sourcesHtml = `
@@ -663,8 +663,8 @@ function renderDocuments() { grid.innerHTML = state.documents.map(doc => { const extRaw = (doc.extension || '').replace('.', '').toLowerCase(); const extLabel = extRaw ? extRaw.toUpperCase() : 'UNKNOWN'; - const docIcon = EXT_ICONS[extRaw] || '📄'; - const uploaderLabel = doc.uploader_username ? `👤 ${escapeHtml(doc.uploader_username)}` : ''; + const docIcon = EXT_ICONS[extRaw] || ''; + const uploaderLabel = doc.uploader_username ? `Uploader: ${escapeHtml(doc.uploader_username)}` : ''; return `
@@ -682,12 +682,12 @@ function renderDocuments() { data-doc-title="${escapeHtml(doc.title || 'Untitled')}" data-doc-ext="${extRaw}" onclick="openDocModal(this.dataset.docId, this.dataset.docTitle, this.dataset.docExt)"> - 👁 Open + Open ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) ? ` ` : ''} @@ -747,7 +747,7 @@ window.openDocAccessModal = async function(docId, currentGroupsRaw) { modal.innerHTML = `
-

⚙️ Document Access

+

Document Access

@@ -777,7 +777,7 @@ window.openDocAccessModal = async function(docId, currentGroupsRaw) { `}
- ${docAccessSelectedGroups.size === 0 ? '🌐 Документ публичный' : `🔒 Доступ ограничен: ${docAccessSelectedGroups.size} гр.`} + ${docAccessSelectedGroups.size === 0 ? 'Документ публичный' : `Доступ ограничен: ${docAccessSelectedGroups.size} гр.`}
@@ -798,8 +798,8 @@ window.toggleDocAccessGroup = function(groupId, isChecked) { const statusEl = document.getElementById('doc-access-status'); if (statusEl) { statusEl.textContent = docAccessSelectedGroups.size === 0 - ? '🌐 Документ публичный' - : `🔒 Доступ ограничен: ${docAccessSelectedGroups.size} гр.`; + ? 'Документ публичный' + : `Доступ ограничен: ${docAccessSelectedGroups.size} гр.`; } } @@ -892,7 +892,7 @@ async function openUploadModal(file = null) {