diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..920fa42 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.venv +venv +__pycache__ +*.pyc +.git +.env +storage +alembic/versions/*.pyc diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..8e17951 --- /dev/null +++ b/.env.local @@ -0,0 +1,2 @@ +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/docs_db +SOLR_URL=http://localhost:8983/solr/chunks \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1baaeb9..f7bda04 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +bin/ +include/ +lib/ +*lib64* target/ -*.so -_rels/ -docProps/ -media/ -word/ -*.xml +share/ +.env* +pyvenv.cfg +tree.txt +*__pycache__* +uploads/* +report/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..75c91fa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Установка системных зависимостей +RUN apt-get update && apt-get install -y gcc libpq-dev && rm -rf /var/lib/apt/lists/* + +# Копирование requirements и установка +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Копирование всего приложения +COPY ./app /app +COPY ./scripts /scripts + +# Создание директории для кэша моделей +RUN mkdir -p /app/model_cache + +# Установка PYTHONPATH +ENV PYTHONPATH=/app + +EXPOSE 8000 + +# Запуск: ждём БД, создаём таблицы, запускаем uvicorn +CMD ["sh", "-c", "python /scripts/wait_for_db.py && python /scripts/init_db.py && uvicorn app.main:app --host 0.0.0.0 --port 8000"] \ No newline at end of file diff --git a/README.md b/README.md index 50f05ea..3daa98b 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,148 @@ -# Docs search - +# Корпоративная RAG-система для работы с приватными документами -## Зависимости и их установка -### Ubuntu +Docs Search - это корпоративная система для семантического поиска и ИИ-анализа документов. Проект использует FastAPI для бэкенда, Qdrant для векторного поиска, PostgreSQL для метаданных и пользователей, а также локальную языковую модель (`cointegrated/rubert-tiny2`) для создания эмбеддингов. + +## Системные требования + +- **ОС:** Linux (Ubuntu/Debian, Arch Linux и др.) +- **Среда:** Python 3.10+ +- **Инфраструктура:** Docker и Docker Compose (для запуска баз данных PostgreSQL и Qdrant) + +--- + +## Установка и первый запуск + +Данная инструкция описывает развёртывание проекта для локальной разработки и тестирования, где базы данных работают в Docker-контейнерах, а сам FastAPI-сервер запускается напрямую в виртуальном окружении. + +### 1. Установка системных зависимостей + +Вам потребуются инструменты для компиляции и библиотеки OCR (Tesseract): + +**Ubuntu/Debian:** ```bash sudo apt update sudo apt install -y build-essential pkg-config clang llvm-dev libclang-dev \ libleptonica-dev libtesseract-dev tesseract-ocr \ - tesseract-ocr-rus tesseract-ocr-eng python3 python3-pip + tesseract-ocr-rus tesseract-ocr-eng python3 python3-pip python3-venv ``` -### Arch Linux +**Arch Linux:** ```bash sudo pacman -Syu --needed --noconfirm build-essential pkgconf clang llvm \ leptonica tesseract tesseract-data-rus \ tesseract-data-eng python-pip ``` -- maturin: - ```bash - # Запускаете .venv - pip install maturin - ``` -## Как запускать? (работа только в .venv окружении) -- Билдим rust либу - ```bash - cd parser - maturin develop - ``` -- Запускаем python - ```bash - python main.py - ``` +### 2. Настройка виртуального окружения (Python) + +Перейдите в директорию проекта, создайте и активируйте виртуальное окружение: + +```bash +python3 -m venv .venv +source .venv/bin/activate +``` + +Установите все необходимые зависимости: +```bash +pip install -r requirements.txt +``` + +### 3. Конфигурация переменных окружения (`.env`) + +Для работы приложению требуются ключи и настройки баз данных. Создайте в корне проекта файл `.env` со следующим содержимым: + +```ini +# Доступы к PostgreSQL +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=docs_db +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/docs_db + +# Секретные ключи для JWT +JWT_SECRET_KEY=your_super_secret_key_here +JWT_REFRESH_SECRET_KEY=your_super_refresh_secret_here +JWT_ALGORITHM=HS256 +JWT_ACCESS_EXPIRE_MINUTES=60 +JWT_REFRESH_EXPIRE_DAYS=7 + +# Учетные данные суперадминистратора +ADMIN_USERNAME=... +ADMIN_PASSWORD=... + +# API Ключи для ИИ чата (если используются GigaChat/DeepSeek) +GIGACHAT_API_KEY=... +DEEPSEEK_API_KEY=... + +# Интеграция локальной LLM через Ollama (опционально) +OLLAMA_HOST=http://localhost:11434 +OLLAMA_MODEL=llama3 +``` + +*(Обязательно замените секретные ключи на надёжные значения в рабочей среде).* + +### 4. Запуск баз данных (Docker) + +Запустите PostgreSQL и Qdrant в фоне с помощью Docker Compose: + +```bash +docker compose up -d +``` +Эта команда создаст контейнеры `docs_postgres` (порт 5432) и `docs_qdrant` (порт 6333) и сохранит их данные в Docker Volumes, чтобы они не исчезли после перезапуска. + +### 5. Применение миграций схемы БД + +Чтобы создать необходимые таблицы в базе данных PostgreSQL, выполните миграции Alembic. +*(Убедитесь, что виртуальное окружение `.venv` активировано!)* + +```bash +export $(grep -v '^#' .env | xargs) +alembic upgrade head +``` + +### 6. Запуск Backend сервера + +Теперь всё готово для запуска основного приложения: + +```bash +# Экспортируем переменные окружения и запускаем FastAPI +export $(grep -v '^#' .env | xargs) +uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +``` + +--- + +## Использование системы + +- **Веб-интерфейс:** Откройте в браузере [http://localhost:8000/ui/](http://localhost:8000/ui/). +- **Вход в систему:** Используйте логин `admin` и пароль `admin1234` (указанные в `.env`), чтобы получить права администратора. +- **API Документация (Swagger):** Доступна по адресу [http://localhost:8000/docs](http://localhost:8000/docs). + +*Примечание: При первом запуске сервер скачает NLP-модель (`cointegrated/rubert-tiny2`), что может занять от нескольких секунд до пары минут в зависимости от скорости интернет-соединения.* + +--- + +## Разработка: Работа с миграциями (Alembic) + +Alembic используется для управления схемой базы данных. Это позволяет легко синхронизировать изменения таблиц. + +**Создать новую миграцию** после изменения схемы в файле `app/models.py`: +```bash +alembic revision --autogenerate -m "описание_изменений" +``` + +**Применить все ожидающие миграции:** +```bash +alembic upgrade head +``` + +**Откатить последнюю миграцию:** +```bash +alembic downgrade -1 +``` + +**Проверить текущее состояние:** +```bash +alembic current # Текущая версия +alembic check # Есть ли незафиксированные изменения +alembic history # История миграций +``` diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..29f1c75 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..e0d0858 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..280218d --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,97 @@ +import os +import asyncio +from logging.config import fileConfig +from dotenv import load_dotenv + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from alembic import context + +# Загружаем переменные из .env (нужно для команд `alembic` из терминала) +load_dotenv() + +# Alembic Config object +config = context.config + +# Настройка логирования из alembic.ini +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# -------------------------------------------------------------- +# Подключаем все модели, чтобы autogenerate видел изменения +# -------------------------------------------------------------- +from app.database import Base # noqa: E402 +import app.models # noqa: F401, E402 - важно: просто импортируем, чтобы модели зарегистрировались + +target_metadata = Base.metadata + +# -------------------------------------------------------------- +# Берём DATABASE_URL из переменной окружения. +# Alembic не умеет работать с asyncpg напрямую, поэтому +# заменяем драйвер: asyncpg → psycopg2 (синхронный) для offline, +# и используем async_engine_from_config для online. +# -------------------------------------------------------------- +def get_url() -> str: + url = os.environ.get("DATABASE_URL", "") + if not url: + raise RuntimeError("DATABASE_URL environment variable is not set") + # Для async-движка оставляем asyncpg; alembic использует его через run_sync + return url + + +def run_migrations_offline() -> None: + """Offline mode: генерирует SQL-скрипт без подключения к БД.""" + # В offline-режиме используем синхронный URL (psycopg2) + url = get_url().replace("postgresql+asyncpg://", "postgresql://") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """Online mode: подключается к БД и применяет миграции.""" + # Переопределяем URL из env для async engine + configuration = config.get_section(config.config_ini_section, {}) + configuration["sqlalchemy.url"] = get_url() + + connectable = async_engine_from_config( + configuration, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Online mode entry point.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/3cad579bf02c_initial_schema.py b/alembic/versions/3cad579bf02c_initial_schema.py new file mode 100644 index 0000000..80f17f2 --- /dev/null +++ b/alembic/versions/3cad579bf02c_initial_schema.py @@ -0,0 +1,79 @@ +"""initial_schema + +Revision ID: 3cad579bf02c +Revises: +Create Date: 2026-06-14 00:11:48.436585 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = '3cad579bf02c' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create initial schema: users, documents, chunks + indexes.""" + + op.create_table( + 'users', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column('username', sa.String(), nullable=False, unique=True), + sa.Column('hashed_password', sa.String(), nullable=False), + sa.Column('role', sa.String(), nullable=True, server_default='user'), + ) + + op.create_table( + 'documents', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column('title', sa.String(), nullable=False), + sa.Column('author', sa.String(), nullable=True), + sa.Column('uploader_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('users.id'), nullable=True), + sa.Column('upload_date', sa.DateTime(), nullable=True), + sa.Column('last_edited', sa.DateTime(), nullable=True), + sa.Column('extension', sa.String(), nullable=True), + sa.Column('size_bytes', sa.BigInteger(), nullable=True), + sa.Column('description', sa.String(), nullable=True), + sa.Column('file_path', sa.String(), nullable=True), + sa.Column('is_available_to', postgresql.JSONB(), nullable=True), + ) + + op.create_table( + 'chunks', + sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, nullable=False), + sa.Column('document_id', postgresql.UUID(as_uuid=True), + sa.ForeignKey('documents.id', ondelete='CASCADE'), nullable=True), + sa.Column('chunk_index', sa.Integer(), nullable=True), + sa.Column('text', sa.String(), nullable=True), + sa.Column('keywords', postgresql.JSONB(), nullable=True), + sa.Column('language', sa.String(), nullable=True), + sa.Column('start_char', sa.Integer(), nullable=True), + sa.Column('end_char', sa.Integer(), nullable=True), + ) + + # Индексы + op.create_index('idx_chunks_document_id', 'chunks', ['document_id']) + op.create_index( + 'idx_chunks_keywords_gin', + 'chunks', + ['keywords'], + postgresql_using='gin', + postgresql_ops={'keywords': 'jsonb_ops'}, + ) + + +def downgrade() -> None: + """Drop all tables and indexes.""" + op.drop_index('idx_chunks_keywords_gin', table_name='chunks') + op.drop_index('idx_chunks_document_id', table_name='chunks') + op.drop_table('chunks') + op.drop_table('documents') + op.drop_table('users') diff --git a/alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py b/alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py new file mode 100644 index 0000000..80dcb42 --- /dev/null +++ b/alembic/versions/d3bf9fb0bfec_add_groups_and_available_to_groups.py @@ -0,0 +1,51 @@ +"""add_groups_and_available_to_groups + +Revision ID: d3bf9fb0bfec +Revises: 3cad579bf02c +Create Date: 2026-06-14 01:50:32.654089 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'd3bf9fb0bfec' +down_revision: Union[str, Sequence[str], None] = '3cad579bf02c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('groups', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.String(), nullable=True), + sa.Column('created_by', sa.UUID(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['users.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('user_groups', + sa.Column('user_id', sa.UUID(), nullable=False), + sa.Column('group_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['group_id'], ['groups.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id', 'group_id') + ) + op.add_column('documents', sa.Column('available_to_groups', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('documents', 'available_to_groups') + op.drop_table('user_groups') + op.drop_table('groups') + # ### end Alembic commands ### diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..eeee602 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +# Пустой файл, чтобы Python считал app пакетом \ No newline at end of file diff --git a/app/__pycache__/__init__.cpython-314.pyc b/app/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..803d10d Binary files /dev/null and b/app/__pycache__/__init__.cpython-314.pyc differ diff --git a/app/__pycache__/auth.cpython-314.pyc b/app/__pycache__/auth.cpython-314.pyc new file mode 100644 index 0000000..7fa8211 Binary files /dev/null and b/app/__pycache__/auth.cpython-314.pyc differ diff --git a/app/__pycache__/database.cpython-314.pyc b/app/__pycache__/database.cpython-314.pyc new file mode 100644 index 0000000..1911996 Binary files /dev/null and b/app/__pycache__/database.cpython-314.pyc differ diff --git a/app/__pycache__/document_processor.cpython-314.pyc b/app/__pycache__/document_processor.cpython-314.pyc new file mode 100644 index 0000000..40dce9d Binary files /dev/null and b/app/__pycache__/document_processor.cpython-314.pyc differ diff --git a/app/__pycache__/documents.cpython-314.pyc b/app/__pycache__/documents.cpython-314.pyc new file mode 100644 index 0000000..9717a0f Binary files /dev/null and b/app/__pycache__/documents.cpython-314.pyc differ diff --git a/app/__pycache__/embeddings.cpython-314.pyc b/app/__pycache__/embeddings.cpython-314.pyc new file mode 100644 index 0000000..0e878e1 Binary files /dev/null and b/app/__pycache__/embeddings.cpython-314.pyc differ diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000..bf08ffc Binary files /dev/null and b/app/__pycache__/main.cpython-314.pyc differ diff --git a/app/__pycache__/models.cpython-314.pyc b/app/__pycache__/models.cpython-314.pyc new file mode 100644 index 0000000..991792e Binary files /dev/null and b/app/__pycache__/models.cpython-314.pyc differ diff --git a/app/__pycache__/qdrant_client.cpython-314.pyc b/app/__pycache__/qdrant_client.cpython-314.pyc new file mode 100644 index 0000000..9a5a925 Binary files /dev/null and b/app/__pycache__/qdrant_client.cpython-314.pyc differ diff --git a/app/__pycache__/schemas.cpython-314.pyc b/app/__pycache__/schemas.cpython-314.pyc new file mode 100644 index 0000000..4501992 Binary files /dev/null and b/app/__pycache__/schemas.cpython-314.pyc differ diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 0000000..b792acd --- /dev/null +++ b/app/auth.py @@ -0,0 +1,320 @@ +""" +Аутентификация: JWT (access + refresh), хэширование паролей, +зависимости FastAPI и роутер /auth. +""" +import os +import logging +from datetime import datetime, timedelta, timezone + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm +from jose import JWTError, jwt +import bcrypt +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select + +from app.database import get_db +from app.models import User +from app.schemas import ( + UserRegisterRequest, + TokenResponse, + RefreshTokenRequest, + UserResponse, + UserSearchResult, +) +import uuid + +logger = logging.getLogger(__name__) + +# ------------------------------------------ +# Конфигурация из переменных окружения +# ------------------------------------------ + +SECRET_KEY = os.getenv("JWT_SECRET_KEY") +REFRESH_SECRET_KEY = os.getenv("JWT_REFRESH_SECRET_KEY") +ALGORITHM = os.getenv("JWT_ALGORITHM", "HS256") +ACCESS_EXPIRE_MINUTES = int(os.getenv("JWT_ACCESS_EXPIRE_MINUTES", "60")) +REFRESH_EXPIRE_DAYS = int(os.getenv("JWT_REFRESH_EXPIRE_DAYS", "7")) + +ADMIN_USERNAME = os.getenv("ADMIN_USERNAME", "") +ADMIN_PASSWORD = os.getenv("ADMIN_PASSWORD", "") + +if not SECRET_KEY: + logger.warning( + "JWT_SECRET_KEY not set in environment! Using insecure dev fallback." + ) + SECRET_KEY = "dev-secret-key-please-set-in-env" + +if not REFRESH_SECRET_KEY: + logger.warning( + "JWT_REFRESH_SECRET_KEY not set in environment! Using insecure dev fallback." + ) + REFRESH_SECRET_KEY = "dev-refresh-secret-please-set-in-env" + +# ------------------------------------------ +# Хэширование паролей (bcrypt напрямую, без passlib) +# ------------------------------------------ + +def get_password_hash(password: str) -> str: + """Возвращает bcrypt-хэш пароля.""" + password_bytes = password.encode("utf-8") + salt = bcrypt.gensalt() + return bcrypt.hashpw(password_bytes, salt).decode("utf-8") + + +def verify_password(plain_password: str, hashed_password: str) -> bool: + """Сравнивает открытый пароль с bcrypt-хэшем.""" + return bcrypt.checkpw( + plain_password.encode("utf-8"), + hashed_password.encode("utf-8"), + ) + + +# ------------------------------------------ +# JWT - создание и декодирование +# ------------------------------------------ + +def create_access_token(data: dict) -> str: + """ + Создаёт короткоживущий access-токен. + TTL = JWT_ACCESS_EXPIRE_MINUTES (по умолчанию 60 мин). + """ + payload = data.copy() + expire = datetime.now(timezone.utc) + timedelta(minutes=ACCESS_EXPIRE_MINUTES) + payload.update({"exp": expire, "type": "access"}) + return jwt.encode(payload, SECRET_KEY, algorithm=ALGORITHM) + + +def create_refresh_token(data: dict) -> str: + """ + Создаёт долгоживущий refresh-токен. + TTL = JWT_REFRESH_EXPIRE_DAYS (по умолчанию 7 дней). + """ + payload = data.copy() + expire = datetime.now(timezone.utc) + timedelta(days=REFRESH_EXPIRE_DAYS) + payload.update({"exp": expire, "type": "refresh"}) + return jwt.encode(payload, REFRESH_SECRET_KEY, algorithm=ALGORITHM) + + +def decode_access_token(token: str) -> dict: + """ + Декодирует и валидирует access-токен. + Выбрасывает JWTError при невалидном или истёкшем токене. + """ + payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + if payload.get("type") != "access": + raise JWTError("Wrong token type") + return payload + + +def decode_refresh_token(token: str) -> dict: + """ + Декодирует и валидирует refresh-токен. + Выбрасывает JWTError при невалидном или истёкшем токене. + """ + payload = jwt.decode(token, REFRESH_SECRET_KEY, algorithms=[ALGORITHM]) + if payload.get("type") != "refresh": + raise JWTError("Wrong token type") + return payload + + +# ------------------------------------------ +# FastAPI Dependencies +# ------------------------------------------ + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth/login") + +_CREDENTIALS_EXCEPTION = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, +) + + +async def get_current_user( + token: str = Depends(oauth2_scheme), + db: AsyncSession = Depends(get_db), +) -> User: + """ + Dependency: извлекает текущего пользователя из access-токена. + Выбрасывает 401, если токен невалиден или пользователь не найден. + """ + try: + payload = decode_access_token(token) + username: str = payload.get("sub") + if username is None: + raise _CREDENTIALS_EXCEPTION + except JWTError: + raise _CREDENTIALS_EXCEPTION + + result = await db.execute(select(User).where(User.username == username)) + user = result.scalar_one_or_none() + if user is None: + raise _CREDENTIALS_EXCEPTION + return user + + +async def get_current_admin( + current_user: User = Depends(get_current_user), +) -> User: + """ + Dependency: проверяет, что текущий пользователь - admin. + Выбрасывает 403, если нет. + """ + if current_user.role != "admin": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin access required", + ) + return current_user + + +# ------------------------------------------ +# Роутер /auth +# ------------------------------------------ + +router = APIRouter(prefix="/auth", tags=["auth"]) + + +@router.post("/register", response_model=UserResponse, status_code=201) +async def register( + body: UserRegisterRequest, + db: AsyncSession = Depends(get_db), +): + """ + Регистрация нового пользователя. + - Если username совпадает с ADMIN_USERNAME и пароль совпадает с ADMIN_PASSWORD - + роль будет 'admin'. + - Иначе - роль 'user'. + """ + # Проверяем уникальность имени + result = await db.execute(select(User).where(User.username == body.username)) + if result.scalar_one_or_none() is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Username already taken", + ) + + # Определяем роль + role = "user" + if ( + ADMIN_USERNAME + and body.username == ADMIN_USERNAME + and body.password == ADMIN_PASSWORD + ): + role = "admin" + + user = User( + username=body.username, + hashed_password=get_password_hash(body.password), + role=role, + ) + db.add(user) + await db.commit() + await db.refresh(user) + + logger.info(f"Registered user '{user.username}' with role '{user.role}'") + return user + + +@router.post("/login", response_model=TokenResponse) +async def login( + form: OAuth2PasswordRequestForm = Depends(), + db: AsyncSession = Depends(get_db), +): + """ + Логин по username + password (OAuth2 form). + Возвращает пару access_token + refresh_token. + """ + result = await db.execute(select(User).where(User.username == form.username)) + user = result.scalar_one_or_none() + + if user is None or not verify_password(form.password, user.hashed_password): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Bearer"}, + ) + + token_data = {"sub": user.username} + return TokenResponse( + access_token=create_access_token(token_data), + refresh_token=create_refresh_token(token_data), + ) + + +@router.post("/refresh", response_model=TokenResponse) +async def refresh_tokens( + body: RefreshTokenRequest, + db: AsyncSession = Depends(get_db), +): + """ + Обновление токенов по refresh_token. + Возвращает новую пару access_token + refresh_token. + """ + try: + payload = decode_refresh_token(body.refresh_token) + username: str = payload.get("sub") + if not username: + raise JWTError("No subject") + except JWTError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid refresh token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Убедимся, что пользователь ещё существует в БД + result = await db.execute(select(User).where(User.username == username)) + if result.scalar_one_or_none() is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found", + ) + + token_data = {"sub": username} + return TokenResponse( + access_token=create_access_token(token_data), + refresh_token=create_refresh_token(token_data), + ) + + +@router.get("/me", response_model=UserResponse) +async def get_me(current_user: User = Depends(get_current_user)): + """Возвращает данные текущего авторизованного пользователя.""" + return current_user + + +@router.get("/users", response_model=list[UserSearchResult]) +async def search_users( + q: str | None = None, + group_id: str | None = None, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Поиск пользователей. Только для admin. + q - фильтр по username (ILIKE) + group_id - вернуть только членов определённой группы + """ + if current_user.role != "admin": + raise HTTPException(status_code=403, detail="Admin access required") + + from app.models import user_groups + + stmt = select(User) + if group_id: + try: + gid = uuid.UUID(group_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid group_id format") + stmt = ( + stmt + .join(user_groups, User.id == user_groups.c.user_id) + .where(user_groups.c.group_id == gid) + ) + if q: + stmt = stmt.where(User.username.ilike(f"%{q}%")) + + result = await db.execute(stmt.order_by(User.username).limit(50)) + return result.scalars().all() diff --git a/app/chat.py b/app/chat.py new file mode 100644 index 0000000..0336ded --- /dev/null +++ b/app/chat.py @@ -0,0 +1,113 @@ +import os +import json +import logging +import httpx +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.database import get_db +from app.models import User +from app.schemas import ChatRequest +from app.auth import get_current_user +from app.search import hybrid_search + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/chat", tags=["chat"]) + +OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://localhost:11434") +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3") + +async def generate_chat_response(query: str, current_user: User, db: AsyncSession): + # 1. Поиск контекста + try: + search_results = await hybrid_search( + query=query, + current_user=current_user, + db=db, + top_k=5 + ) + except Exception as e: + logger.error(f"Error during search: {e}") + yield json.dumps({"type": "error", "content": "Failed to retrieve context."}) + "\n" + return + + # Отправка источников первым пакетом + sources = [ + { + "document_id": r.document_id, + "document_title": r.document_title, + "extension": r.extension, + "score": r.score + } for r in search_results + ] + # Дедупликация источников для UI + unique_sources = list({s["document_id"]: s for s in sources}.values()) + + yield json.dumps({"type": "sources", "sources": unique_sources}) + "\n" + + # 2. Формирование промпта + if not search_results: + context_text = "Нет релевантных документов для этого запроса." + else: + context_text = "\n\n".join( + f"--- Документ: {r.document_title} ---\n{r.text}" + for r in search_results + ) + + system_prompt = ( + "Вы - корпоративный ИИ-ассистент. Ваша задача - отвечать на вопросы пользователя, " + "основываясь ТОЛЬКО на предоставленном контексте из корпоративных документов. " + "Если в контексте нет ответа на вопрос, честно скажите, что не знаете. " + "Не придумывайте информацию. Отвечайте на русском языке.\n\n" + f"КОНТЕКСТ:\n{context_text}" + ) + + payload = { + "model": OLLAMA_MODEL, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": query} + ], + "stream": True + } + + # 3. Отправка запроса в Ollama и стриминг + try: + async with httpx.AsyncClient() as client: + async with client.stream("POST", f"{OLLAMA_HOST}/api/chat", json=payload, timeout=60.0) as response: + if response.status_code != 200: + error_text = await response.aread() + yield json.dumps({"type": "error", "content": f"Ollama error: {response.status_code} {error_text.decode()}"}) + "\n" + return + + async for chunk in response.aiter_lines(): + if chunk: + try: + data = json.loads(chunk) + if "message" in data and "content" in data["message"]: + yield json.dumps({ + "type": "content", + "content": data["message"]["content"] + }) + "\n" + except json.JSONDecodeError: + logger.error(f"Failed to parse Ollama chunk: {chunk}") + except httpx.RequestError as e: + logger.error(f"Error communicating with Ollama: {e}") + yield json.dumps({"type": "error", "content": "Ошибка связи с Ollama. Проверьте, запущен ли сервер."}) + "\n" + + +@router.post("") +async def chat_endpoint( + request: ChatRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + if not request.query.strip(): + raise HTTPException(status_code=400, detail="Query cannot be empty.") + + return StreamingResponse( + generate_chat_response(request.query, current_user, db), + media_type="application/x-ndjson" + ) diff --git a/app/database.py b/app/database.py new file mode 100644 index 0000000..cbafc83 --- /dev/null +++ b/app/database.py @@ -0,0 +1,15 @@ +import os +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker +from sqlalchemy.orm import declarative_base + +# Используем localhost вместо postgres (так как запускаем локально) +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/docs_db") + +engine = create_async_engine(DATABASE_URL, echo=True) +AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession) + +Base = declarative_base() + +async def get_db(): + async with AsyncSessionLocal() as session: + yield session \ No newline at end of file diff --git a/app/document_processor.py b/app/document_processor.py new file mode 100644 index 0000000..4aefb9a --- /dev/null +++ b/app/document_processor.py @@ -0,0 +1,69 @@ +import uuid +from app.database import AsyncSessionLocal +from app.models import Chunk +from app.qdrant_client import index_chunk +from app.embeddings import get_embedding +from app.keyword_extraction.paragraph_processor import ParagraphProcessor +from app.keyword_extraction.keyword_processor import KeywordProcessor +from app.keyword_extraction.language_detection import detect_language_simple +import logging + +logger = logging.getLogger(__name__) + +chunker = ParagraphProcessor(max_chunk_size=1000, overlap=200) +kw_processor = KeywordProcessor() + +import docs_parser + +async def process_document(document_id: str, file_path: str, metadata: dict = None): + try: + raw_text, _ = docs_parser.extract_text(file_path) + except Exception as e: + logger.error(f"Failed to read file {file_path}: {e}") + return + + chunks_data = chunker.process_paragraph(raw_text) + if not chunks_data: + logger.warning(f"No chunks generated for document {document_id}") + return + + async with AsyncSessionLocal() as db: + for idx, chunk_info in enumerate(chunks_data): + chunk_text = chunk_info['text'] + language = detect_language_simple(chunk_text) + + keywords_with_scores = kw_processor.extract_keywords_from_text( + chunk_text, language, top_n=5 + ) + keywords = [kw for kw, _ in keywords_with_scores] + + embedding = get_embedding(chunk_text) + + chunk_id = uuid.uuid4() + chunk = Chunk( + id=chunk_id, + document_id=uuid.UUID(document_id), + chunk_index=idx, + text=chunk_text, + keywords=keywords, + language=language, + start_char=chunk_info.get('start', 0), + end_char=chunk_info.get('end', 0) + ) + db.add(chunk) + await db.flush() + + success = index_chunk( + chunk_id=str(chunk_id), + document_id=document_id, + chunk_index=idx, + text=chunk_text, + keywords=keywords, + language=language, + embedding=embedding + ) + if not success: + logger.warning(f"Qdrant indexing failed for chunk {chunk_id}") + + await db.commit() + logger.info(f"Document {document_id} processed: {len(chunks_data)} chunks") \ No newline at end of file diff --git a/app/documents.py b/app/documents.py new file mode 100644 index 0000000..3db6d83 --- /dev/null +++ b/app/documents.py @@ -0,0 +1,403 @@ +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, BackgroundTasks, Query +from fastapi.responses import FileResponse +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy import or_, and_, cast, String, func +import mimetypes +import uuid +import os + +from app.database import get_db +from app.models import User, Document, Chunk, user_groups +from app.schemas import ( + DocumentResponse, DocumentUploadResponse, DocumentUpdateRequest, + PaginatedDocuments, + ChunkResponse, SearchRequest, SearchResultItem, +) +from app.auth import get_current_user +from app.document_processor import process_document +from app.qdrant_client import delete_chunks_by_document +from app.search import hybrid_search + +router = APIRouter(prefix="/documents", tags=["documents"]) + + +UPLOAD_DIR = "uploads/raw" +os.makedirs(UPLOAD_DIR, exist_ok=True) + + +# ------------------------------------------ +# Вспомогательная функция проверки доступа +# ------------------------------------------ + +async def check_document_access(document: Document, current_user: User, db: AsyncSession) -> bool: + """ + Возвращает True если пользователь имеет доступ к документу. + Логика: + - Владелец документа → всегда есть доступ. + - Admin → всегда есть доступ. + - Оба списка пусты (is_available_to=None и available_to_groups=None) → публичный, доступ есть. + - ID пользователя в is_available_to → доступ есть. + - Пользователь состоит в группе из available_to_groups → доступ есть. + """ + if str(document.uploader_id) == str(current_user.id): + return True + if current_user.role == "admin": + return True + + both_empty = ( + (not document.is_available_to or document.is_available_to == 'null') + and (not document.available_to_groups or document.available_to_groups == 'null') + ) + if both_empty: + return True + + if document.is_available_to and str(current_user.id) in document.is_available_to: + return True + + if document.available_to_groups: + user_group_ids = await _get_user_group_ids(current_user.id, db) + if any(gid in document.available_to_groups for gid in user_group_ids): + return True + + return False + + +async def _get_user_group_ids(user_id: uuid.UUID, db: AsyncSession) -> list[str]: + result = await db.execute( + select(user_groups.c.group_id).where(user_groups.c.user_id == user_id) + ) + return [str(row[0]) for row in result.fetchall()] + + +# ------------------------------------------ +# Upload +# ------------------------------------------ + +@router.post("/upload", response_model=DocumentUploadResponse) +async def upload_document( + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + title: str | None = Form(None), + author: str | None = Form(None), + description: str | None = Form(None), + is_available_to: str | None = Form(None), # строка UUID через запятую + available_to_groups: str | None = Form(None), # строка UUID через запятую + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + document_id = uuid.uuid4() + + # Парсинг is_available_to + available_users: list[str] = [] + if is_available_to: + for uid in is_available_to.split(","): + uid = uid.strip() + if uid: + try: + uuid.UUID(uid) + available_users.append(uid) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid user UUID: {uid}") + + # Парсинг available_to_groups + # Обычный пользователь может назначать только группы, в которых сам состоит + available_groups: list[str] = [] + if available_to_groups: + raw_groups = [g.strip() for g in available_to_groups.split(",") if g.strip()] + if current_user.role != "admin": + user_group_ids = await _get_user_group_ids(current_user.id, db) + # Фильтруем - только свои группы + raw_groups = [g for g in raw_groups if g in user_group_ids] + for gid in raw_groups: + try: + uuid.UUID(gid) + available_groups.append(gid) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid group UUID: {gid}") + + # Расширение и сохранение файла + filename = file.filename or "" + _, ext = os.path.splitext(filename) + extension = ext.lstrip(".").lower() if ext else None + + file_path = os.path.join(UPLOAD_DIR, f"{document_id}.{extension}" if extension else str(document_id)) + try: + with open(file_path, "wb") as f: + content = await file.read() + f.write(content) + size_bytes = len(content) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}") + + # Запись в БД + document = Document( + id=document_id, + title=title or filename, + author=author, + uploader_id=current_user.id, + extension=extension, + size_bytes=size_bytes, + description=description, + file_path=file_path, + is_available_to=available_users if available_users else None, + available_to_groups=available_groups if available_groups else None, + ) + + db.add(document) + await db.commit() + + background_tasks.add_task(process_document, document_id=str(document_id), file_path=file_path) + + return DocumentUploadResponse(document_id=document_id, status="processing") + + +# ------------------------------------------ +# Helpers +# ------------------------------------------ + +async def _enrich_with_uploader(document: Document, db: AsyncSession) -> DocumentResponse: + """Добавляет имя загрузчика к ответу документа.""" + uploader_username = None + if document.uploader_id: + result = await db.execute(select(User).where(User.id == document.uploader_id)) + uploader = result.scalar_one_or_none() + if uploader: + uploader_username = uploader.username + return DocumentResponse( + id=document.id, + title=document.title, + author=document.author, + uploader_id=document.uploader_id, + uploader_username=uploader_username, + upload_date=document.upload_date, + last_edited=document.last_edited, + extension=document.extension, + size_bytes=document.size_bytes, + description=document.description, + is_available_to=document.is_available_to, + available_to_groups=document.available_to_groups, + ) + + +# ------------------------------------------ +# List / Get +# ------------------------------------------ + +@router.get("", response_model=PaginatedDocuments) +async def list_documents( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), + search: str = Query(default="", description="Filter by title substring"), + page: int = Query(default=1, ge=1, description="Page number (1-indexed)"), + page_size: int = Query(default=12, ge=1, le=100, description="Items per page"), +): + # Базовое условие доступности + if current_user.role == "admin": + access_filter = True # без ограничений + base_query = select(Document) + else: + user_group_ids = await _get_user_group_ids(current_user.id, db) + group_conditions = [ + Document.available_to_groups.has_key(gid) + for gid in user_group_ids + ] + access_where = or_( + Document.uploader_id == current_user.id, + and_( + or_(Document.is_available_to.is_(None), cast(Document.is_available_to, String).in_(('null', '[]'))), + or_(Document.available_to_groups.is_(None), cast(Document.available_to_groups, String).in_(('null', '[]'))) + ), + Document.is_available_to.has_key(str(current_user.id)), + *group_conditions, + ) + base_query = select(Document).where(access_where) + + # Фильтр по названию + if search: + base_query = base_query.where(Document.title.ilike(f"%{search}%")) + + # Подсчёт общего количества + count_result = await db.execute( + select(func.count()).select_from(base_query.subquery()) + ) + total = count_result.scalar() or 0 + + # Пагинация + offset = (page - 1) * page_size + paged_query = base_query.order_by(Document.upload_date.desc()).offset(offset).limit(page_size) + result = await db.execute(paged_query) + docs = result.scalars().all() + + pages = max(1, -(-total // page_size)) # ceiling division + + items = [await _enrich_with_uploader(doc, db) for doc in docs] + return PaginatedDocuments(items=items, total=total, page=page, pages=pages, page_size=page_size) + + +@router.get("/{doc_id}", response_model=DocumentResponse) +async def get_document( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if not await check_document_access(document, current_user, db): + raise HTTPException(status_code=403, detail="Access denied") + + return await _enrich_with_uploader(document, db) + + +# ------------------------------------------ +# Update +# ------------------------------------------ + +@router.put("/{doc_id}", response_model=DocumentResponse) +async def update_document( + doc_id: uuid.UUID, + request: DocumentUpdateRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if current_user.role != "admin" and document.uploader_id != current_user.id: + raise HTTPException(status_code=403, detail="Only the owner or an admin can update this document") + + if request.title is not None: + document.title = request.title + if request.author is not None: + document.author = request.author + if request.description is not None: + document.description = request.description + if request.is_available_to is not None: + document.is_available_to = [str(u) for u in request.is_available_to] if request.is_available_to else None + if request.available_to_groups is not None: + # Проверяем права: обычный пользователь - только свои группы + if current_user.role != "admin": + user_group_ids = await _get_user_group_ids(current_user.id, db) + filtered = [str(g) for g in request.available_to_groups if str(g) in user_group_ids] + else: + filtered = [str(g) for g in request.available_to_groups] + document.available_to_groups = filtered if filtered else None + + await db.commit() + await db.refresh(document) + return await _enrich_with_uploader(document, db) + + +# ------------------------------------------ +# Delete +# ------------------------------------------ + +@router.delete("/{doc_id}") +async def delete_document( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db) +): + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + if current_user.role != "admin" and document.uploader_id != current_user.id: + raise HTTPException(status_code=403, detail="Only the owner or an admin can delete this document") + + if document.file_path and os.path.exists(document.file_path): + try: + os.remove(document.file_path) + except Exception as e: + print(f"Failed to delete file {document.file_path}: {e}") + + delete_chunks_by_document(str(doc_id)) + await db.delete(document) + await db.commit() + + return {"status": "deleted"} + + +# ------------------------------------------ +# Search +# ------------------------------------------ + +@router.post("/search", response_model=list[SearchResultItem]) +async def search_documents( + request: SearchRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Гибридный поиск по документам пользователя. + Комбинирует семантический поиск (Qdrant) и поиск по ключевым словам (PostgreSQL). + """ + if not request.query.strip(): + raise HTTPException(status_code=400, detail="Query must not be empty") + results = await hybrid_search( + query=request.query, + current_user=current_user, + db=db, + top_k=request.top_k, + ) + return results + + +# ------------------------------------------ +# File Content (Preview / Download) +# ------------------------------------------ + +INLINE_EXTENSIONS = {"pdf", "txt", "png", "jpg", "jpeg", "gif", "webp", "svg"} + + +@router.get("/{doc_id}/content") +async def get_document_content( + doc_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Возвращает оригинальный файл документа. + - PDF, TXT, изображения - отдаются inline (отображаются в браузере). + - DOCX, XLSX, PPTX и пр. - отдаются как вложение (скачивание). + Доступ проверяется по правам пользователя, включая группы. + """ + result = await db.execute(select(Document).where(Document.id == doc_id)) + document = result.scalar_one_or_none() + + if document is None: + raise HTTPException(status_code=404, detail="Document not found") + + if not await check_document_access(document, current_user, db): + raise HTTPException(status_code=403, detail="Access denied") + + if not document.file_path or not os.path.exists(document.file_path): + raise HTTPException(status_code=404, detail="File not found on disk") + + ext = (document.extension or "").lstrip(".").lower() + mime_type, _ = mimetypes.guess_type(document.file_path) + if not mime_type: + mime_type = "application/octet-stream" + + disposition = "inline" if ext in INLINE_EXTENSIONS else "attachment" + filename = os.path.basename(document.file_path) + + from urllib.parse import quote + + return FileResponse( + path=document.file_path, + media_type=mime_type, + filename=filename, + content_disposition_type=disposition, + headers={ + "X-Document-Title": quote(document.title or filename), + }, + ) diff --git a/app/embeddings.py b/app/embeddings.py new file mode 100644 index 0000000..0be36a8 --- /dev/null +++ b/app/embeddings.py @@ -0,0 +1,30 @@ +from sentence_transformers import SentenceTransformer +import logging + +logger = logging.getLogger(__name__) + +# Глобальная переменная для модели (загружается один раз при старте) +_model = None + +def load_model(): + """Загружает модель эмбеддингов (ленивая загрузка)""" + global _model + if _model is None: + logger.info("Loading embedding model: cointegrated/rubert-tiny2") + _model = SentenceTransformer('cointegrated/rubert-tiny2') + logger.info("Model loaded successfully") + return _model + +def get_embedding(text: str) -> list[float]: + """ + Возвращает вектор эмбеддинга для текста. + Размерность: 312. + """ + model = load_model() + # convert_to_numpy=True возвращает numpy array, .tolist() превращает в list + embedding = model.encode(text, convert_to_numpy=True).tolist() + return embedding + +def get_embedding_dimension() -> int: + """Возвращает размерность эмбеддинга (312 для rubert-tiny2)""" + return 312 \ No newline at end of file diff --git a/app/groups.py b/app/groups.py new file mode 100644 index 0000000..7ed41f3 --- /dev/null +++ b/app/groups.py @@ -0,0 +1,204 @@ +""" +управление группами доступа. +Создание/удаление групп и управление участниками - только для admin. +Просмотр списка групп - для всех авторизованных пользователей. +""" +import uuid +import logging +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy import delete, func + +from app.database import get_db +from app.models import User, Group, user_groups +from app.schemas import GroupCreate, GroupResponse, GroupMemberAdd, UserSearchResult +from app.auth import get_current_user + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/groups", tags=["groups"]) + + +# ------------------------------------------ +# Вспомогательные функции +# ------------------------------------------ + +def require_admin(current_user: User = Depends(get_current_user)) -> User: + if current_user.role != "admin": + raise HTTPException(status_code=403, detail="Admin access required") + return current_user + + +async def get_user_group_ids(user_id: uuid.UUID, db: AsyncSession) -> list[str]: + """Возвращает список строковых ID групп, в которых состоит пользователь.""" + result = await db.execute( + select(user_groups.c.group_id).where(user_groups.c.user_id == user_id) + ) + return [str(row[0]) for row in result.fetchall()] + + +# ------------------------------------------ +# Группы CRUD +# ------------------------------------------ + +@router.get("", response_model=list[GroupResponse]) +async def list_groups( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """ + Список групп. + Администраторы видят все группы. + Обычные пользователи - только те группы, в которых они состоят. + """ + if current_user.role == "admin": + stmt = select(Group).order_by(Group.name) + else: + stmt = ( + select(Group) + .join(user_groups, Group.id == user_groups.c.group_id) + .where(user_groups.c.user_id == current_user.id) + .order_by(Group.name) + ) + + result = await db.execute(stmt) + groups = result.scalars().all() + + # Подсчитываем количество участников для каждой группы + group_list = [] + for g in groups: + count_result = await db.execute( + select(func.count()).select_from(user_groups).where(user_groups.c.group_id == g.id) + ) + count = count_result.scalar() or 0 + group_list.append(GroupResponse( + id=g.id, + name=g.name, + description=g.description, + created_at=g.created_at, + member_count=count, + )) + return group_list + + +@router.post("", response_model=GroupResponse) +async def create_group( + data: GroupCreate, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Создать новую группу. Только для admin.""" + existing = await db.execute(select(Group).where(Group.name == data.name)) + if existing.scalar_one_or_none(): + raise HTTPException(status_code=400, detail="Группа с таким именем уже существует") + + group = Group( + id=uuid.uuid4(), + name=data.name, + description=data.description, + created_by=current_user.id, + ) + db.add(group) + await db.commit() + await db.refresh(group) + return GroupResponse( + id=group.id, + name=group.name, + description=group.description, + created_at=group.created_at, + member_count=0, + ) + + +@router.delete("/{group_id}") +async def delete_group( + group_id: uuid.UUID, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Удалить группу. Только для admin.""" + result = await db.execute(select(Group).where(Group.id == group_id)) + group = result.scalar_one_or_none() + if not group: + raise HTTPException(status_code=404, detail="Group not found") + await db.delete(group) + await db.commit() + return {"status": "deleted"} + + +# ------------------------------------------ +# Участники групп +# ------------------------------------------ + +@router.get("/{group_id}/members", response_model=list[UserSearchResult]) +async def get_group_members( + group_id: uuid.UUID, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Список участников группы.""" + group_result = await db.execute(select(Group).where(Group.id == group_id)) + if not group_result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="Group not found") + + stmt = ( + select(User) + .join(user_groups, User.id == user_groups.c.user_id) + .where(user_groups.c.group_id == group_id) + .order_by(User.username) + ) + result = await db.execute(stmt) + return result.scalars().all() + + +@router.post("/{group_id}/members") +async def add_members( + group_id: uuid.UUID, + data: GroupMemberAdd, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Добавить пользователей в группу. Только для admin.""" + group_result = await db.execute(select(Group).where(Group.id == group_id)) + if not group_result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="Group not found") + + added = [] + for user_id in data.user_ids: + user_result = await db.execute(select(User).where(User.id == user_id)) + if not user_result.scalar_one_or_none(): + continue + + existing = await db.execute( + select(user_groups).where( + user_groups.c.user_id == user_id, + user_groups.c.group_id == group_id, + ) + ) + if not existing.first(): + await db.execute( + user_groups.insert().values(user_id=user_id, group_id=group_id) + ) + added.append(str(user_id)) + + await db.commit() + return {"added": added} + + +@router.delete("/{group_id}/members/{user_id}") +async def remove_member( + group_id: uuid.UUID, + user_id: uuid.UUID, + current_user: User = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Убрать пользователя из группы. Только для admin.""" + await db.execute( + delete(user_groups).where( + user_groups.c.user_id == user_id, + user_groups.c.group_id == group_id, + ) + ) + await db.commit() + return {"status": "removed"} diff --git a/app/keyword_extraction/README.md b/app/keyword_extraction/README.md new file mode 100644 index 0000000..0e2faa4 --- /dev/null +++ b/app/keyword_extraction/README.md @@ -0,0 +1,10 @@ +# Multilingual Semantic Keyword Set +A tool for extracting and managing keywords with semantic deduplication. + +## Supported languages +Russian and English. + +## What it does +It extracts keywords from a given text. + +For English it does so quite straightforward. As for Russian, it normalizes collocations, making every word of the collocation singular, masculine (if verb or adjective), nominative case (e.g. 'нейронные сети' becomes 'нейронный сеть'). \ No newline at end of file diff --git a/app/keyword_extraction/__init__.py b/app/keyword_extraction/__init__.py new file mode 100644 index 0000000..b70a069 --- /dev/null +++ b/app/keyword_extraction/__init__.py @@ -0,0 +1,22 @@ +""" +Multilingual Semantic Keyword Set +A tool for extracting and managing keywords with semantic deduplication. +Supported languages: Russian and English. +""" + +from .keyword_set import MultilingualKeywordSet +from .collocation_deduplicator import CollocationDeduplicator +from .language_detection import detect_language_simple +from .paragraph_processor import ParagraphProcessor +from .keyword_processor import KeywordProcessor +from .collocation_extractor import CollocationExtractor + +__version__ = "1.0.2" +__all__ = [ + 'MultilingualKeywordSet', + 'CollocationDeduplicator', + 'detect_language_simple', + 'ParagraphProcessor', + 'KeywordProcessor', + 'CollocationExtractor' +] \ No newline at end of file diff --git a/app/keyword_extraction/__pycache__/__init__.cpython-314.pyc b/app/keyword_extraction/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..4453209 Binary files /dev/null and b/app/keyword_extraction/__pycache__/__init__.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/collocation_deduplicator.cpython-314.pyc b/app/keyword_extraction/__pycache__/collocation_deduplicator.cpython-314.pyc new file mode 100644 index 0000000..a45d6bf Binary files /dev/null and b/app/keyword_extraction/__pycache__/collocation_deduplicator.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/collocation_extractor.cpython-314.pyc b/app/keyword_extraction/__pycache__/collocation_extractor.cpython-314.pyc new file mode 100644 index 0000000..a2d76a9 Binary files /dev/null and b/app/keyword_extraction/__pycache__/collocation_extractor.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/keyword_processor.cpython-314.pyc b/app/keyword_extraction/__pycache__/keyword_processor.cpython-314.pyc new file mode 100644 index 0000000..7fb52c0 Binary files /dev/null and b/app/keyword_extraction/__pycache__/keyword_processor.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/keyword_set.cpython-314.pyc b/app/keyword_extraction/__pycache__/keyword_set.cpython-314.pyc new file mode 100644 index 0000000..acf0438 Binary files /dev/null and b/app/keyword_extraction/__pycache__/keyword_set.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/language_detection.cpython-314.pyc b/app/keyword_extraction/__pycache__/language_detection.cpython-314.pyc new file mode 100644 index 0000000..38ddb1a Binary files /dev/null and b/app/keyword_extraction/__pycache__/language_detection.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/lemmatization.cpython-314.pyc b/app/keyword_extraction/__pycache__/lemmatization.cpython-314.pyc new file mode 100644 index 0000000..e687515 Binary files /dev/null and b/app/keyword_extraction/__pycache__/lemmatization.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/paragraph_processor.cpython-314.pyc b/app/keyword_extraction/__pycache__/paragraph_processor.cpython-314.pyc new file mode 100644 index 0000000..8fcb67b Binary files /dev/null and b/app/keyword_extraction/__pycache__/paragraph_processor.cpython-314.pyc differ diff --git a/app/keyword_extraction/__pycache__/stopwords.cpython-314.pyc b/app/keyword_extraction/__pycache__/stopwords.cpython-314.pyc new file mode 100644 index 0000000..f6ea52b Binary files /dev/null and b/app/keyword_extraction/__pycache__/stopwords.cpython-314.pyc differ diff --git a/app/keyword_extraction/collocation_deduplicator.py b/app/keyword_extraction/collocation_deduplicator.py new file mode 100644 index 0000000..b42b34e --- /dev/null +++ b/app/keyword_extraction/collocation_deduplicator.py @@ -0,0 +1,118 @@ +""" +Collocation deduplication utilities. +Handles removal of overlapping and redundant collocations. +""" + +from typing import List, Tuple, Set +import re + +class CollocationDeduplicator: + """ + Handles deduplication of overlapping collocations. + """ + + @staticmethod + def tokenize_phrase(phrase: str) -> List[str]: + """Split a phrase into tokens (words).""" + return phrase.lower().split() + + @staticmethod + def calculate_overlap(phrase1: str, phrase2: str) -> Tuple[int, List[str]]: + """ + Calculate overlapping words between two phrases. + Returns (overlap_count, overlapping_words). + """ + tokens1 = set(CollocationDeduplicator.tokenize_phrase(phrase1)) + tokens2 = set(CollocationDeduplicator.tokenize_phrase(phrase2)) + + overlap = tokens1.intersection(tokens2) + return len(overlap), list(overlap) + + @staticmethod + def is_subsumed(longer: str, shorter: str) -> bool: + """ + Check if shorter phrase is completely contained within longer phrase. + Example: "neural network" is subsumed by "deep neural network" + """ + tokens_longer = CollocationDeduplicator.tokenize_phrase(longer) + tokens_shorter = CollocationDeduplicator.tokenize_phrase(shorter) + + # Check if all tokens of shorter are in longer (in order) + long_idx = 0 + for short_token in tokens_shorter: + found = False + while long_idx < len(tokens_longer): + if tokens_longer[long_idx] == short_token: + found = True + long_idx += 1 + break + long_idx += 1 + if not found: + return False + return True + + + @staticmethod + def remove_overlapping_collocations(collocations: List[Tuple[str, float]], + overlap_threshold: float = 0.6) -> List[Tuple[str, float]]: + """ + Remove overlapping collocations, keeping the most specific (longer) ones. + + Args: + collocations: List of (phrase, score) tuples + overlap_threshold: Minimum overlap ratio to consider as overlapping + (e.g., 0.6 means 60% of words overlap) + + Returns: + Deduplicated list of collocations + """ + if not collocations: + return [] + + # Sort by length (longest first) and then by score + sorted_collocations = sorted(collocations, + key=lambda x: (len(x[0].split()), x[1]), + reverse=True) + + kept = [] + kept_phrases = [] + + for phrase, score in sorted_collocations: + tokens_phrase = set(CollocationDeduplicator.tokenize_phrase(phrase)) + phrase_len = len(tokens_phrase) + + should_keep = True + + # Check against already kept phrases + for kept_phrase in kept_phrases: + tokens_kept = set(CollocationDeduplicator.tokenize_phrase(kept_phrase)) + kept_len = len(tokens_kept) + + # Calculate overlap + overlap_count = len(tokens_phrase.intersection(tokens_kept)) + overlap_ratio = overlap_count / min(phrase_len, kept_len) + + # If significant overlap exists + if overlap_ratio >= overlap_threshold: + # If current phrase is longer, replace the kept one + if phrase_len > kept_len: + # Remove the kept phrase (will be replaced) + continue + else: + # Current phrase is subsumed by kept phrase + should_keep = False + break + + if should_keep: + # Check if this phrase subsumes any kept phrases + new_kept = [] + for kept_phrase in kept_phrases: + if not CollocationDeduplicator.is_subsumed(phrase, kept_phrase): + new_kept.append(kept_phrase) + new_kept.append(phrase) + kept_phrases = new_kept + kept.append((phrase, score)) + + # Sort by score again for output + kept.sort(key=lambda x: x[1], reverse=True) + return kept \ No newline at end of file diff --git a/app/keyword_extraction/collocation_extractor.py b/app/keyword_extraction/collocation_extractor.py new file mode 100644 index 0000000..95e4a03 --- /dev/null +++ b/app/keyword_extraction/collocation_extractor.py @@ -0,0 +1,195 @@ +""" +Collocation extraction utilities for multilingual text. +""" + +import re +from collections import Counter +from typing import List, Tuple, Set +from .stopwords import get_stopwords, is_stopword +from .lemmatization import is_content_word +from .collocation_deduplicator import CollocationDeduplicator + +# Try to import NLTK +try: + import nltk + from nltk.collocations import BigramCollocationFinder, TrigramCollocationFinder + from nltk.metrics import BigramAssocMeasures, TrigramAssocMeasures + + nltk_data_downloaded = False + try: + nltk.data.find('tokenizers/punkt') + nltk.data.find('corpora/stopwords') + nltk_data_downloaded = True + except LookupError: + print("Downloading required NLTK data...") + try: + nltk.download('punkt', quiet=True) + nltk.download('punkt_tab', quiet=True) + nltk.download('stopwords', quiet=True) + nltk_data_downloaded = True + print("NLTK data downloaded successfully") + except Exception as e: + print(f"Could not download NLTK data: {e}") + nltk_data_downloaded = False + + NLTK_AVAILABLE = nltk_data_downloaded + +except ImportError: + print("NLTK not installed. Using fallback methods.") + NLTK_AVAILABLE = False + nltk = None + + +class CollocationExtractor: + """Extracts and processes collocations from text.""" + + def __init__(self, max_ngram_size: int = 3, deduplicator=None): + self.max_ngram_size = max_ngram_size + self.deduplicator = deduplicator or CollocationDeduplicator() + + def extract_collocations_fallback(self, text: str, language: str = 'en', + top_n: int = 10, verbose: bool = False) -> List[Tuple[str, float]]: + """Fallback method for extracting collocations.""" + from .stopwords import get_punctuation_pattern + punct_pattern = get_punctuation_pattern(language) + clean_text = re.sub(punct_pattern, ' ', text.lower()) + words = clean_text.split() + + collocations = [] + + # Стоп-слова, которые не должны быть в коллокациях + stopwords = get_stopwords(language) + bad_patterns = {'это', 'он', 'она', 'оно', 'они', 'этот', 'эта', 'это', 'эти'} + + for n in range(2, self.max_ngram_size + 1): + for i in range(len(words) - n + 1): + ngram_words = words[i:i+n] + ngram = ' '.join(ngram_words) + + # Пропускаем коллокации, начинающиеся с местоимений или предлогов + first_word = ngram_words[0] + if first_word in stopwords or first_word in bad_patterns: + continue + + # Пропускаем коллокации, заканчивающиеся на предлоги + last_word = ngram_words[-1] + if last_word in {'о', 'об', 'в', 'на', 'для', 'с', 'к', 'у', 'по', 'за'}: + continue + + # Проверяем, что коллокация содержит хотя бы одно знаменательное слово + has_content = False + for w in ngram_words: + if is_content_word(w) if language == 'ru' else len(w) > 3: + has_content = True + break + + if not has_content: + continue + + # Считаем частоту + freq = sum(1 for j in range(len(words) - n + 1) + if ' '.join(words[j:j+n]) == ngram) + + if freq >= 1: + # Вес коллокации: частота + длина + score = freq * len(ngram) / 50 + + # Бонус за длину + if n == 2: + score *= 1.2 + elif n == 3: + score *= 1.5 + + collocations.append((ngram, score)) + + # Удаляем дубликаты + unique_collocations = {} + for colloc, score in collocations: + if colloc not in unique_collocations or score > unique_collocations[colloc]: + unique_collocations[colloc] = score + + # Отладочный вывод + if verbose: + print(f"\n[DEBUG] Filtered collocations (after cleaning):") + for colloc, score in list(unique_collocations.items())[:15]: + if verbose: + print(f" - '{colloc}' (score: {score:.3f})") + + sorted_collocations = sorted(unique_collocations.items(), + key=lambda x: x[1], reverse=True) + + return sorted_collocations[:top_n] + + def extract_collocations_nltk(self, text: str, language: str = 'en', + top_n: int = 10) -> List[Tuple[str, float]]: + """Extract collocations using NLTK.""" + if not NLTK_AVAILABLE: + return self.extract_collocations_fallback(text, language, top_n) + + try: + from nltk.corpus import stopwords + + try: + nltk_stopwords = set(stopwords.words(language)) + except: + nltk_stopwords = get_stopwords(language) + + tokens = nltk.word_tokenize(text.lower()) + + filtered_tokens = [token for token in tokens + if token.isalnum() and token not in nltk_stopwords and len(token) > 2] + + if len(filtered_tokens) < 2: + return [] + + collocations = [] + + if self.max_ngram_size >= 2: + try: + bigram_finder = BigramCollocationFinder.from_words(filtered_tokens) + bigram_finder.apply_freq_filter(1) + bigrams = bigram_finder.nbest(BigramAssocMeasures.pmi, min(top_n, 20)) + for bigram in bigrams: + collocation = ' '.join(bigram) + score = len(collocation) / 50 + collocations.append((collocation, score)) + except: + pass + + if self.max_ngram_size >= 3: + try: + trigram_finder = TrigramCollocationFinder.from_words(filtered_tokens) + trigram_finder.apply_freq_filter(1) + trigrams = trigram_finder.nbest(TrigramAssocMeasures.pmi, min(top_n, 20)) + for trigram in trigrams: + collocation = ' '.join(trigram) + score = len(collocation) / 50 + collocations.append((collocation, score)) + except: + pass + + unique_collocations = {} + for colloc, score in collocations: + if colloc not in unique_collocations or score > unique_collocations[colloc]: + unique_collocations[colloc] = score + + sorted_collocations = sorted(unique_collocations.items(), + key=lambda x: x[1], reverse=True) + + # Apply deduplication + deduplicated = self.deduplicator.remove_overlapping_collocations( + sorted_collocations, 0.6 + ) + + return deduplicated[:top_n] + + except Exception as e: + return self.extract_collocations_fallback(text, language, top_n) + + def extract_collocations(self, text: str, language: str = 'en', + top_n: int = 10, verbose: bool = False) -> List[Tuple[str, float]]: + """Extract collocations from text with deduplication.""" + if NLTK_AVAILABLE: + return self.extract_collocations_nltk(text, language, top_n) + else: + return self.extract_collocations_fallback(text, language, top_n, verbose) \ No newline at end of file diff --git a/app/keyword_extraction/example_usage.py b/app/keyword_extraction/example_usage.py new file mode 100644 index 0000000..2004a7c --- /dev/null +++ b/app/keyword_extraction/example_usage.py @@ -0,0 +1,52 @@ +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +os.environ['TRANSFORMERS_OFFLINE'] = '1' +os.environ['HF_HUB_OFFLINE'] = '1' +os.environ['HF_DATASETS_OFFLINE'] = '1' + +from keyword_extraction import MultilingualKeywordSet + +model_cache_path = './model_cache2' + +def enter_example(): + os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1' + os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '1' + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + # Это длинный абзац о машинном обучении. Он охватывает несколько тем, включая нейронные сети, глубокие нейронные сети и сверточные нейронные сети. Цель состоит в том, чтобы извлечь значимые ключевые слова, такие как алгоритмы машинного обучения и методы глубокого обучения, из длинных текстов. + # Современные технологии искусственного интеллекта и машинного обучения стремительно трансформируют различные отрасли промышленности и науки. Глубокие нейронные сети, которые представляют собой многослойные архитектуры с десятками и даже сотнями слоёв, демонстрируют впечатляющие результаты в распознавании образов, обработке естественного языка и компьютерном зрении. Свёрточные нейронные сети, впервые предложенные Яном Лекуном в конце двадцатого века, стали основой для большинства современных систем компьютерного зрения, включая распознавание лиц, автоматическую аннотацию изображений и диагностику медицинских снимков. Рекуррентные нейронные сети и их более совершенные варианты, такие как LSTM и GRU, успешно применяются для анализа последовательных данных, включая временные ряды, тексты на естественном языке и геномные последовательности. Методы глубокого обучения с подкреплением позволили создать алгоритмы, которые превосходят человека в таких сложных играх, как го, шахматы и различные видеоигры от компании Blizzard. Трансформеры, архитектура которых была представлена в знаменитой статье Attention Is All You Need, произвели настоящую революцию в области обработки естественного языка, что привело к созданию таких моделей, как BERT, GPT, T5 и многих других. Эти модели используют механизм внимания, который позволяет эффективно обрабатывать длинные последовательности и улавливать сложные зависимости между элементами текста. + # В последние годы наблюдается значительный прогресс в области квантовых вычислений и их потенциального применения для решения сложных оптимизационных задач. Квантовые процессоры, основанные на сверхпроводящих кубитах и захваченных ионах, постепенно набирают вычислительную мощность, приближаясь к так называемому квантовому превосходству. Исследователи из Google впервые продемонстрировали квантовое превосходство в 2019 году, когда их 53-кубитный процессор Sycamore выполнил специфическую задачу за 200 секунд, на решение которой у самого мощного классического суперкомпьютера потребовалось бы около десяти тысяч лет. Однако критики отмечают, что эта задача была специально подобрана для квантового компьютера и не имеет практического применения. Тем не менее, учёные активно работают над созданием отказоустойчивых квантовых компьютеров с коррекцией ошибок, что является одним из главных препятствий на пути к практическому использованию квантовых алгоритмов. Параллельно развиваются и гибридные подходы, сочетающие классические и квантовые вычисления, которые уже сейчас могут применяться для оптимизации логистических цепочек, моделирования молекулярных структур в фармацевтике и решения других прикладных задач. + # Цифровая трансформация бизнеса в эпоху четвёртой промышленной революции требует от компаний не только внедрения новых технологий, но и пересмотра фундаментальных бизнес-моделей и организационных структур. Большие данные, аналитика в реальном времени и предиктивное моделирование становятся ключевыми инструментами для принятия стратегических решений в условиях высокой неопределённости и турбулентности рынков. Пандемия коронавируса значительно ускорила процессы цифровизации во всех секторах экономики, от розничной торговли и образования до здравоохранения и государственного управления. Компании, которые раньше игнорировали необходимость развития онлайн-каналов продаж и удалённых форматов работы, оказались в крайне уязвимом положении. В то же время лидеры цифровой экономики, такие как Amazon, Microsoft и китайские технологические гиганты, смогли не только сохранить, но и значительно нарастить свои рыночные позиции. Интересно отметить, что внедрение технологий искусственного интеллекта и роботизации процессов пока что приводит к созданию новых рабочих мест и трансформации существующих профессий, а не к массовой безработице, как прогнозировали некоторые пессимистичные эксперты в начале двухтысячных годов. + # Прорывы в области генной инженерии и персонализированной медицины открывают беспрецедентные возможности для лечения ранее неизлечимых заболеваний. Технология редактирования генома CRISPR-Cas9, открытая Эммануэль Шарпантье и Дженнифер Дудной, позволяет вносить точные изменения в ДНК живых организмов, открывая путь к исправлению генетических мутаций, вызывающих наследственные заболевания. Уже проводятся клинические испытания методов терапии серповидноклеточной анемии, муковисцидоза и некоторых форм наследственной слепоты с использованием технологий генного редактирования. Параллельно развиваются методы иммунотерапии рака, включая CAR-T-клеточную терапию, при которой иммунные клетки пациента генетически модифицируются для более эффективного распознавания и уничтожения злокачественных опухолей. В онкологии также активно применяются алгоритмы машинного обучения для анализа медицинских изображений, что позволяет на ранних стадиях выявлять злокачественные новообразования и значительно повышает точность диагностики. Модели глубокого обучения способны различать доброкачественные и злокачественные образования на маммограммах и компьютерных томограммах с точностью, сопоставимой или даже превосходящей опытных радиологов. + # Глобальное изменение климата и антропогенное воздействие на окружающую среду становятся одними из главных вызовов двадцать первого века, требующих безотлагательных и скоординированных действий на международном уровне. Концентрация парниковых газов в атмосфере достигла рекордных значений за последние восемьсот тысяч лет, что приводит к повышению средней температуры на планете, таянию ледников и учащению экстремальных погодных явлений. Участившиеся лесные пожары в Австралии и Сибири, разрушительные ураганы в Атлантике, наводнения в Европе и Юго-Восточной Азии напрямую связаны с климатическими изменениями. Учёные предупреждают, что если не принять решительных мер по сокращению выбросов углекислого газа и других парниковых газов, последствия будут катастрофическими для экосистем и человеческой цивилизации. В ответ на эту угрозу многие страны взяли на себя обязательства по достижению углеродной нейтральности к середине столетия, что предполагает масштабный переход к возобновляемым источникам энергии, электрификацию транспорта и повышение энергоэффективности промышленности и строительства. + # И конечно же никто никогда не мог предположить, что старые методы и подходы окажутся полностью бесполезными в новой реальности. Однако тот самый наш опыт показывает, что не всё так просто и однозначно в этом вопросе. Мы долго обсуждали эту проблему и в итоге пришли к выводу, что нужно действовать незамедлительно. Сначала они попытались решить всё с помощью обычных алгоритмов, но потом поняли, что это был довольно наивный подход. В конце концов, как говорится, настоящие профессионалы всегда ищут нестандартные решения и не боятся экспериментировать. Именно поэтому было принято решение об использовании самых современных нейросетевых архитектур для обработки поступающих данных. + # This is a longer paragraph about machine learning. It covers multiple topics including neural networks, deep neural networks, and convolutional neural networks. The goal is to extract meaningful keywords like machine learning algorithms and deep learning methods from longer texts. + keyword_set = MultilingualKeywordSet( + initial_keywords={ + 'en': [], + 'ru': [] + }, + similarity_threshold=0.75, + collocation_overlap_threshold=0.6, + cache_folder=model_cache_path, + embedding_model='cointegrated/rubert-tiny2', + use_collocations=True, + max_chunk_size=500 + ) + + print("Enter your text: ") + s = input() + + result = keyword_set.process_text(s) + + if 'keywords' in result and result['keywords']: + for i, word in enumerate(result['keywords']): + print(f"{i + 1}: {word}") + else: + print("No keywords found.") + +if __name__ == "__main__": + enter_example() \ No newline at end of file diff --git a/app/keyword_extraction/example_usage.saving_to_file.py b/app/keyword_extraction/example_usage.saving_to_file.py new file mode 100644 index 0000000..3ae6a96 --- /dev/null +++ b/app/keyword_extraction/example_usage.saving_to_file.py @@ -0,0 +1,153 @@ +""" +Example usage of the Multilingual Keyword Set. +""" + +import sys +import os + +# Add parent directory to path if running directly +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from keyword_extraction import MultilingualKeywordSet + +model_cache_path = './model_cache' + +def example_usage(): + os.environ['TRANSFORMERS_OFFLINE'] = '1' + print("MULTILINGUAL SEMANTIC KEYWORD SET (No langdetect)") + print("=" * 70) + + initial_keywords = { + 'en': ["machine learning", "neural network", "deep learning"], + 'ru': ["машинное обучение", "нейронная сеть", "глубокое обучение"] + } + + keyword_set = MultilingualKeywordSet( + initial_keywords=initial_keywords, + similarity_threshold=0.75, + collocation_overlap_threshold=0.6, + cache_folder=model_cache_path, + use_collocations=True, + max_chunk_size=500 + ) + + print("\nInitial keyword set:") + keyword_set.display_keyword_set() + + # Test texts with overlapping phrases + texts = [ + # English sentences + "Deep neural networks are revolutionizing artificial intelligence", + "Глубокие нейронные сети революционизируют искусственный интеллект", + "Support vector machines are used for classification tasks", + "Методы опорных векторов используются для задач классификации", + + # Long English paragraph with overlapping phrases + """ + This is a longer paragraph about machine learning. + It covers multiple topics including neural networks, deep neural networks, + and convolutional neural networks. The goal is to extract meaningful keywords + like machine learning algorithms and deep learning methods from longer texts. + """, + + # Long Russian paragraph with overlapping phrases + """ + Это длинный абзац о машинном обучении. + Он охватывает несколько тем, включая нейронные сети, глубокие нейронные сети + и сверточные нейронные сети. Цель состоит в том, чтобы извлечь значимые ключевые слова, + такие как алгоритмы машинного обучения и методы глубокого обучения, из длинных текстов. + """, + + # Another English text with more overlapping collocations + """ + Convolutional neural networks (CNNs) are a type of neural network architecture. + These deep learning models have revolutionized computer vision tasks. + Transfer learning allows reusing pre-trained neural networks for new tasks. + """ + ] + + print("\n" + "=" * 70) + print("PROCESSING TEXTS WITH OVERLAPPING COLLOCATIONS") + print("=" * 70) + + for i, text in enumerate(texts, 1): + print(f"\n{'#'*70}") + print(f"Text {i}/{len(texts)}") + print(f"{'#'*70}") + keyword_set.process_text(text, top_n=6, verbose=True) + + print("\n" + "=" * 70) + print("FINAL RESULTS") + print("=" * 70) + + # Show final keyword set with frequencies + keyword_set.display_keyword_set(show_frequencies=True) + + # Save keywords to file + keyword_set.save_keywords_to_file('./multilingual_keywords.txt') + """ + # Optional: Show the saved keywords + print("\n" + "=" * 70) + print("SAVED KEYWORDS PREVIEW") + print("=" * 70) + with open('multilingual_keywords.txt', 'r', encoding='utf-8') as f: + lines = f.readlines() + for line in lines[:20]: # Show first 20 lines + print(line.rstrip()) + if len(lines) > 20: + print(f"... and {len(lines) - 20} more lines") + """ + +def simple_example(): + os.environ['TRANSFORMERS_OFFLINE'] = '1' + """Very simple example for quick testing.""" + print("\n" + "=" * 70) + print("SIMPLE QUICK TEST") + print("=" * 70) + + keyword_set = MultilingualKeywordSet( + similarity_threshold=0.7, + collocation_overlap_threshold=0.6, + cache_folder='./simple_cache' + ) + + # Quick English test + keyword_set.process_text("Machine learning is transforming technology", top_n=3) + + # Quick Russian test + keyword_set.process_text("Искусственный интеллект меняет мир", top_n=3) + + # Display results + keyword_set.display_keyword_set() + +def enter_example(): + os.environ['TRANSFORMERS_OFFLINE'] = '1' + # Это длинный абзац о машинном обучении. Он охватывает несколько тем, включая нейронные сети, глубокие нейронные сети и сверточные нейронные сети. Цель состоит в том, чтобы извлечь значимые ключевые слова, такие как алгоритмы машинного обучения и методы глубокого обучения, из длинных текстов. + + keyword_set = MultilingualKeywordSet( + initial_keywords={ + 'en': [], + 'ru': [] + }, + similarity_threshold=0.75, + collocation_overlap_threshold=0.6, + cache_folder=model_cache_path, + use_collocations=True, + max_chunk_size=500 + ) + + print("Enter your text: ") + s = input() + + keyword_set.process_text(s, verbose=True) + + keyword_set.display_keyword_set() + + keyword_set.save_keywords_to_file('./entered_text_keywords.txt') + + +if __name__ == "__main__": + # Run the main example + # example_usage() + + enter_example() \ No newline at end of file diff --git a/app/keyword_extraction/keyword_processor.py b/app/keyword_extraction/keyword_processor.py new file mode 100644 index 0000000..e7d07f8 --- /dev/null +++ b/app/keyword_extraction/keyword_processor.py @@ -0,0 +1,147 @@ +""" +Keyword processing utilities for extraction and filtering. +""" + +import re +from collections import Counter +from typing import List, Tuple, Set +from .stopwords import get_punctuation_pattern, is_stopword +from .lemmatization import lemmatize_russian, lemmatize_russian_collocation, is_function_word +from .language_detection import detect_language_simple + + +class KeywordProcessor: + """Processes keywords: extraction, lemmatization, filtering.""" + + def __init__(self, keybert_model=None): + self.keybert = keybert_model + + def lemmatize_if_necessary(self, keyword: str) -> str: + """Lemmatize keyword if it's Russian.""" + if detect_language_simple(keyword).lower() == 'ru': + return lemmatize_russian_collocation(keyword) + return keyword + + def filter_stopwords(self, words: List[str]) -> List[str]: + """Filter out stopwords from a list of words.""" + filtered = [] + for w in words: + lang = detect_language_simple(w).lower() + lemmatized = self.lemmatize_if_necessary(w) + if not is_stopword(lemmatized, lang) and len(w) > 2: + filtered.append(w) + return filtered + + def extract_simple_keywords(self, text: str, language: str = 'en') -> List[str]: + """Simple keyword extraction as fallback.""" + punct_pattern = get_punctuation_pattern(language) + clean_text = re.sub(punct_pattern, ' ', text.lower()) + words = clean_text.split() + + keywords = self.filter_stopwords(words) + + unique_keywords = [] + seen = set() + for kw in keywords: + if kw not in seen: + seen.add(kw) + unique_keywords.append(kw) + + return unique_keywords[:10] + + def prepare_keyword_for_addition(self, keyword: str, language: str) -> str: + """Prepare keyword before adding to set (lemmatize, normalize).""" + keyword_lower = keyword.lower().strip() + + if language == 'ru': + keyword_lower = lemmatize_russian_collocation(keyword_lower) + + return keyword_lower + + def should_skip_keyword(self, keyword: str, language: str, + existing_keywords: set = None) -> Tuple[bool, str]: + """ + Check if keyword should be skipped. + + Returns: + (should_skip, reason) + """ + keyword_lower = keyword.lower().strip() + + # Check length + if len(keyword_lower) < 3: + return True, "too short" + + # Check if single word is part of existing collocation + if existing_keywords and len(keyword_lower.split()) == 1: + for existing in existing_keywords: + if len(existing.split()) >= 2 and keyword_lower in existing.split(): + return True, f"part of collocation '{existing}'" + + # Check stopwords + words = keyword_lower.split() + stopword_count = sum(1 for w in words if is_stopword(w, language)) + if len(words) > 0 and stopword_count == len(words): + return True, "all words are stopwords" + + # Check function words for Russian + if language == 'ru': + func_count = sum(1 for w in words if is_function_word(w)) + if len(words) > 0 and func_count == len(words): + return True, "all words are function words" + + return False, "" + + def extract_keywords_from_text(self, text: str, language: str = 'en', + top_n: int = 10, verbose: bool = False) -> List[Tuple[str, float]]: + """Extract keywords and collocations from text.""" + all_candidates = [] + from .collocation_extractor import CollocationExtractor + + # Extract collocations + coll_extractor = CollocationExtractor(max_ngram_size=3) + collocations = coll_extractor.extract_collocations(text, language, top_n=top_n * 2, verbose=verbose) + + # Lemmatize collocations + if language == 'ru': + lemmatized_collocs = [] + for phrase, score in collocations: + lemmatized_phrase = lemmatize_russian_collocation(phrase) + words = lemmatized_phrase.split() + if len(words) >= 2: + lemmatized_collocs.append((lemmatized_phrase, score * 2.0)) + if verbose: + print(f"[COLLOC] Found: '{phrase}' -> '{lemmatized_phrase}' (score: {score:.3f})") + all_candidates.extend(lemmatized_collocs) + else: + all_candidates.extend(collocations) + + # Extract single words using simple method (without KeyBERT) + simple_keywords = self.extract_simple_keywords(text, language) + + # Filter words that are already in collocations + collocation_words = set() + for phrase, _ in all_candidates: + for word in phrase.split(): + collocation_words.add(word) + + for word in simple_keywords[:top_n]: + if word not in collocation_words: + all_candidates.append((word, 0.5)) + + # Remove duplicates + unique_candidates = {} + for phrase, score in all_candidates: + if phrase not in unique_candidates or score > unique_candidates[phrase]: + unique_candidates[phrase] = score + + sorted_candidates = sorted(unique_candidates.items(), + key=lambda x: x[1], reverse=True) + + # Deduplicate overlapping collocations + from .collocation_deduplicator import CollocationDeduplicator + deduplicated = CollocationDeduplicator.remove_overlapping_collocations( + sorted_candidates, 0.6 + ) + + return deduplicated[:top_n] \ No newline at end of file diff --git a/app/keyword_extraction/keyword_set.py b/app/keyword_extraction/keyword_set.py new file mode 100644 index 0000000..6fb0b73 --- /dev/null +++ b/app/keyword_extraction/keyword_set.py @@ -0,0 +1,383 @@ +""" +Main keyword set class with multilingual support and semantic deduplication. +""" + +import os +import re +import warnings +from collections import defaultdict +from typing import List, Tuple, Set, Dict, Optional + +from sentence_transformers import SentenceTransformer, util + +from .language_detection import detect_language_simple +from .paragraph_processor import ParagraphProcessor +from .collocation_deduplicator import CollocationDeduplicator +from .keyword_processor import KeywordProcessor +from .lemmatization import lemmatize_russian_collocation + +warnings.filterwarnings('ignore') + +class MultilingualKeywordSet: + """Multilingual keyword set with collocation deduplication.""" + + def __init__(self, + initial_keywords: Dict[str, List[str]] = None, + similarity_threshold: float = 0.75, + collocation_overlap_threshold: float = 0.6, + embedding_model: str = 'cointegrated/rubert-tiny2', + cache_folder: str = './model_cache', + use_collocations: bool = True, + max_ngram_size: int = 3, + max_chunk_size: int = 1000, + chunk_overlap: int = 100): + """ + Initialize multilingual keyword set. + """ + self.similarity_threshold = similarity_threshold + self.collocation_overlap_threshold = collocation_overlap_threshold + self.use_collocations = use_collocations + self.max_ngram_size = max_ngram_size + + self.paragraph_processor = ParagraphProcessor(max_chunk_size, chunk_overlap) + self.deduplicator = CollocationDeduplicator() + + if use_collocations: + print("Collocations enabled") + + # Create cache folder + os.makedirs(cache_folder, exist_ok=True) + + # Load model + print(f"Loading multilingual model from: {embedding_model}") + try: + self.model = SentenceTransformer(embedding_model, cache_folder=cache_folder) + print(f"Model loaded successfully") + except Exception as e: + print(f"Could not load {embedding_model}, falling back to English model...") + self.model = SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L3-v2', + cache_folder=cache_folder) + print(f"English model loaded as fallback") + + # Initialize processors + self.keyword_processor = KeywordProcessor(keybert_model=None) # Will be set after model load + self.keybert = None # Will be initialized later if needed + + # Try to initialize KeyBERT with the model + try: + from keybert import KeyBERT + self.keybert = KeyBERT(model=self.model) + self.keyword_processor.keybert = self.keybert + except Exception as e: + print(f"KeyBERT not available ({e}), using fallback extraction") + self.keybert = None + self.keyword_processor.keybert = None + + # Store keywords per language + self.keywords_by_lang: Dict[str, Set[str]] = {} + self.keyword_embeddings: Dict[str, Dict[str, any]] = {} + self.keyword_frequencies: Dict[str, Dict[str, int]] = {} + self.keyword_metadata: Dict[str, Dict] = {} + + # Add initial keywords + if initial_keywords: + for lang, keywords in initial_keywords.items(): + if lang not in self.keywords_by_lang: + self.keywords_by_lang[lang] = set() + self.keyword_embeddings[lang] = {} + self.keyword_frequencies[lang] = defaultdict(int) + + for keyword in keywords: + self.add_keyword(keyword, lang, check_similarity=False) + + # Public methods + + def add_keyword(self, keyword: str, language: str, check_similarity: bool = True, verbose = False) -> bool: + """Add a keyword to the set.""" + # Prepare keyword + keyword_lower = self.keyword_processor.prepare_keyword_for_addition(keyword, language) + + # Check if should skip + existing_keywords = self.keywords_by_lang.get(language, set()) + should_skip, reason = self.keyword_processor.should_skip_keyword( + keyword_lower, language, existing_keywords + ) + if should_skip: + if verbose: + print(f" Skipped '{keyword}' ({language}) - {reason}") + return False + + # Initialize language storage + if language not in self.keywords_by_lang: + self.keywords_by_lang[language] = set() + self.keyword_embeddings[language] = {} + self.keyword_frequencies[language] = defaultdict(int) + + # Check if already exists + if keyword_lower in self.keywords_by_lang[language]: + self.keyword_frequencies[language][keyword_lower] += 1 + return False + + # Check semantic similarity + if check_similarity and self.keywords_by_lang[language]: + try: + new_embedding = self.model.encode(keyword_lower, convert_to_tensor=True) + + max_similarity = 0 + most_similar_keyword = None + + for existing_keyword, existing_embedding in self.keyword_embeddings[language].items(): + similarity = util.pytorch_cos_sim(new_embedding, existing_embedding).item() + if similarity > max_similarity: + max_similarity = similarity + most_similar_keyword = existing_keyword + + if max_similarity >= self.similarity_threshold: + if verbose: + print(f" Rejected '{keyword}' -> '{keyword_lower}' (similar to '{most_similar_keyword}': {max_similarity:.3f})") + return False + else: + self._add_to_storage(keyword, keyword_lower, language, new_embedding) + if verbose: + print(f" Added: '{keyword}' -> '{keyword_lower}' (max similarity: {max_similarity:.3f})") + return True + except Exception as e: + if verbose: + print(f" Error adding keyword '{keyword}': {e}") + return False + else: + try: + embedding = self.model.encode(keyword_lower, convert_to_tensor=True) + self._add_to_storage(keyword, keyword_lower, language, embedding) + if verbose: + print(f" Added: '{keyword}' -> '{keyword_lower}'") + return True + except Exception as e: + if verbose: + print(f" Error adding keyword '{keyword}': {e}") + return False + + def process_text(self, text: str, top_n: int = -1, verbose: bool = False) -> Dict: + """Process text (sentence or paragraph).""" + # Calculate top_n if not specified + if top_n < 0: + marks = r'[,;.!?]+[\n]+' + marks_num = len(re.split(marks, text)) + top_n = marks_num if marks_num > 1 else int(text.count(' ') / 3) + + language = detect_language_simple(text) + + if verbose: + print(f"\n{'='*70}") + print(f"Processing Text (Language: {language.upper()})") + print(f"{'='*70}") + print(f"Text: {text[:150]}...") + + all_keywords = [] + all_new_keywords = [] + + # Process based on text length + if len(text) > self.paragraph_processor.max_chunk_size: + chunks = self.paragraph_processor.process_paragraph(text, language) + if verbose: + print(f"\nSplit into {len(chunks)} chunk(s)") + + for chunk_info in chunks: + keywords, new_keywords = self._process_chunk(chunk_info, language, top_n, verbose) + all_keywords.extend(keywords) + all_new_keywords.extend(new_keywords) + else: + extracted = self.keyword_processor.extract_keywords_from_text(text, language, top_n, verbose) + + if verbose and extracted: + print(f"\nExtracted keywords (after deduplication):") + for i, (candidate, score) in enumerate(extracted[:5], 1): + print(f" {i}. '{candidate}' (score: {score:.3f})") + + for candidate, score in extracted: + lemmatized = self.keyword_processor.lemmatize_if_necessary(candidate) + if self.add_keyword(candidate, language, check_similarity=True, verbose=verbose): + all_new_keywords.append(lemmatized) + all_keywords.append(lemmatized) + else: + similar = self.find_similar_keywords(lemmatized, 1) + if similar: + all_keywords.append(similar[0][0]) + + # Filter final keywords + filtered_keywords = self.keyword_processor.filter_stopwords(list(set(all_keywords))) + + if verbose: + print(f"\nAdded {len(all_new_keywords)} new keyword(s)") + print(f"Total keywords: {sum(len(kw) for kw in self.keywords_by_lang.values())}") + + return { + 'language': language, + 'keywords_added': all_new_keywords, + 'total_keywords': sum(len(kw) for kw in self.keywords_by_lang.values()), + 'keywords': filtered_keywords + } + + # Helper methods + + def _add_to_storage(self, original: str, normalized: str, language: str, embedding): + """Add keyword to storage.""" + self.keywords_by_lang[language].add(normalized) + self.keyword_embeddings[language][normalized] = embedding + self.keyword_frequencies[language][normalized] = 1 + self.keyword_metadata[normalized] = {'language': language, 'original': original} + + def _process_chunk(self, chunk_info: Dict, language: str, top_n: int, verbose: bool): + """Process a single chunk.""" + if verbose: + print(f"\n{'#'*40}") + print(f"Processing Chunk {chunk_info['chunk_id']}") + print(f"{'#'*40}") + + extracted = self.keyword_processor.extract_keywords_from_text( + chunk_info['text'], language, top_n + ) + + if verbose and extracted: + print(f"\nExtracted candidates:") + for i, (candidate, score) in enumerate(extracted[:5], 1): + print(f" {i}. '{candidate}' (score: {score:.3f})") + + all_keywords = [] + all_new_keywords = [] + + for candidate, score in extracted: + lemmatized = self.keyword_processor.lemmatize_if_necessary(candidate) + if self.add_keyword(candidate, language, check_similarity=True, verbose=verbose): + all_new_keywords.append(lemmatized) + all_keywords.append(lemmatized) + else: + similar = self.find_similar_keywords(lemmatized, 1) + if similar: + all_keywords.append(similar[0][0]) + + return all_keywords, all_new_keywords + + def get_all_keywords(self, language: Optional[str] = None, sort_by_frequency: bool = False) -> Dict[str, List[str]]: + """ + Get all keywords, optionally filtered by language. + """ + if language: + if language in self.keywords_by_lang: + keywords = list(self.keywords_by_lang[language]) + if sort_by_frequency: + keywords.sort(key=lambda x: self.keyword_frequencies[language].get(x, 0), reverse=True) + return {language: keywords} + else: + return {language: []} + else: + result = {} + for lang, keywords in self.keywords_by_lang.items(): + sorted_keywords = list(keywords) + if sort_by_frequency: + sorted_keywords.sort(key=lambda x: self.keyword_frequencies[lang].get(x, 0), reverse=True) + result[lang] = sorted_keywords + return result + + def find_similar_keywords(self, keyword: str, top_n: int = 5) -> List[Tuple[str, float, str]]: + """ + Find similar keywords across all languages. + + Returns: + List of (keyword, similarity_score, language) tuples + """ + keyword_lower = keyword.lower().strip() + + try: + keyword_embedding = self.model.encode(keyword_lower, convert_to_tensor=True) + + similarities = [] + for lang, embeddings in self.keyword_embeddings.items(): + for existing_keyword, existing_embedding in embeddings.items(): + similarity = util.pytorch_cos_sim(keyword_embedding, existing_embedding).item() + similarities.append((existing_keyword, similarity, lang)) + + similarities.sort(key=lambda x: x[1], reverse=True) + return similarities[:top_n] + except Exception as e: + print(f"Error finding similar keywords: {e}") + return [] + + def set_similarity_threshold(self, threshold: float): + """Adjust the similarity threshold.""" + self.similarity_threshold = threshold + print(f"Similarity threshold updated to {threshold}") + + def display_keyword_set(self, show_frequencies: bool = False): + """Display all keywords organized by language.""" + print(f"\nCurrent Multilingual Keyword Set") + print("=" * 60) + + for lang in sorted(self.keywords_by_lang.keys()): + count = len(self.keywords_by_lang[lang]) + print(f"\n{lang.upper()} ({count} keywords):") + print("-" * 40) + + if show_frequencies: + sorted_items = sorted(self.keyword_frequencies[lang].items(), + key=lambda x: x[1], reverse=True) + for i, (kw, freq) in enumerate(sorted_items[:20], 1): + print(f"{i:2d}. {kw:<35} (freq: {freq})") + else: + for i, kw in enumerate(sorted(self.keywords_by_lang[lang])[:20], 1): + print(f"{i:2d}. {kw}") + + if count > 20: + print(f" ... and {count - 20} more") + + print("=" * 60) + + def save_keywords_to_file(self, filename: str): + """Save current keywords to a file.""" + with open(filename, 'w', encoding='utf-8') as f: + f.write("# Multilingual Keyword Set\n") + f.write(f"# Similarity Threshold: {self.similarity_threshold}\n") + f.write(f"# Collocation Overlap Threshold: {self.collocation_overlap_threshold}\n\n") + + for lang in sorted(self.keywords_by_lang.keys()): + f.write(f"\n## {lang.upper()} Keywords:\n") + for kw in sorted(self.keywords_by_lang[lang]): + freq = self.keyword_frequencies[lang].get(kw, 1) + f.write(f"{kw}\t{freq}\t{lang}\n") + + print(f"Keywords saved to {filename}") + + def load_keywords_from_file(self, filename: str): + """Load keywords from a file.""" + if not os.path.exists(filename): + print(f"File {filename} not found") + return + + with open(filename, 'r', encoding='utf-8') as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + if not line or line.startswith('#') or line.startswith('##'): + continue + + parts = line.split('\t') + if len(parts) >= 2: + keyword = parts[0] + freq = int(parts[1]) if len(parts) > 1 else 1 + lang = parts[2] if len(parts) > 2 else 'en' + + if lang not in self.keywords_by_lang: + self.keywords_by_lang[lang] = set() + self.keyword_embeddings[lang] = {} + self.keyword_frequencies[lang] = defaultdict(int) + + if keyword not in self.keywords_by_lang[lang]: + self.keywords_by_lang[lang].add(keyword) + try: + self.keyword_embeddings[lang][keyword] = self.model.encode(keyword, convert_to_tensor=True) + self.keyword_frequencies[lang][keyword] = freq + except Exception as e: + print(f"Warning: Could not load '{keyword}': {e}") + + print(f"Loaded keywords from {filename}") \ No newline at end of file diff --git a/app/keyword_extraction/language_detection.py b/app/keyword_extraction/language_detection.py new file mode 100644 index 0000000..1eff936 --- /dev/null +++ b/app/keyword_extraction/language_detection.py @@ -0,0 +1,61 @@ +""" +Simple language detection for English and Russian. +No external libraries. +""" + +import re +from .stopwords import get_stopwords + +def detect_language_simple(text: str) -> str: + """ + Enhanced language detection using stopwords and character detection. + Returns 'en' for English, 'ru' for Russian. + """ + # Clean the text (remove punctuation, convert to lowercase) + clean_text = re.sub(r'[^\w\s]', ' ', text.lower()) + words = clean_text.split() + + if not words: + return 'en' + + # Method 1: Cyrillic character detection (быстрый, но неточный) + cyrillic_range = r'[а-яА-ЯёЁ]' + cyrillic_chars = len(re.findall(cyrillic_range, text)) + + # Method 2: Stopword-based detection (более точный) + ru_stopwords = get_stopwords('ru') + en_stopwords = get_stopwords('en') + + # Count stopwords from each language + ru_stopword_count = sum(1 for word in words if word in ru_stopwords) + en_stopword_count = sum(1 for word in words if word in en_stopwords) + + # Method 3: Common Russian indicators (для коротких текстов) + russian_indicators = [ + 'это', 'что', 'который', 'также', 'ещё', 'очень', 'можно', + 'нельзя', 'нужно', 'почему', 'потому', 'поэтому', 'итак' + ] + ru_indicator_count = sum(1 for word in words if word in russian_indicators) + + # Decision logic + # If strong Cyrillic presence -> Russian + if cyrillic_chars > len(text) * 0.3: + return 'ru' + + # If strong stopword evidence + if ru_stopword_count > en_stopword_count * 1.5: + return 'ru' + + if en_stopword_count > ru_stopword_count * 1.5: + return 'en' + + # If many Russian function words + if ru_indicator_count >= 2: + return 'ru' + + # Check for Russian-specific patterns + if any(word.endswith(('ться', 'тся', 'щий', 'щая', 'щее')) for word in words): + return 'ru' + + # Default to English + return 'en' \ No newline at end of file diff --git a/app/keyword_extraction/lemmatization.py b/app/keyword_extraction/lemmatization.py new file mode 100644 index 0000000..aba04da --- /dev/null +++ b/app/keyword_extraction/lemmatization.py @@ -0,0 +1,62 @@ +""" +Words lemmatization utilities for Russian. +Lemmatizes words in Russian. +""" +import pymorphy3 + +morph_ru = pymorphy3.MorphAnalyzer() + +def lemmatize_russian(word: str) -> str: + """ + Приводит русское слово к начальной форме. + Если лемматизация не удалась, возвращает исходное слово. + """ + try: + parsed = morph_ru.parse(word)[0] + return parsed.normal_form + except: + return word + +def lemmatize_russian_collocation(phrase: str) -> str: + """ + Лемматизирует всю коллокацию (фразу из нескольких слов). + Пример: "глубокие нейронные сети" -> "глубокий нейронный сеть" + """ + if not phrase or len(phrase.strip()) == 0: + return phrase + + words = phrase.lower().split() + lemmatized_words = [lemmatize_russian(word) for word in words] + return ' '.join(lemmatized_words) + +def is_function_word(word: str) -> bool: + """ + Определяет, является ли слово служебной частью речи. + Возвращает True для слов, которые стоит исключить из ключевых слов. + """ + try: + parsed = morph_ru.parse(word)[0] + pos = parsed.tag.POS + + # Служебные части речи (стоит исключить) + function_positions = {'PREP', 'CONJ', 'PRCL', 'INTJ', 'NPRO'} + + # Также исключаем числительные + if pos == 'NUMR': + return True + + return pos in function_positions + except: + return False + +def is_content_word(word: str) -> bool: + """ + Определяет, является ли слово знаменательной частью речи. + """ + try: + parsed = morph_ru.parse(word)[0] + pos = parsed.tag.POS + content_positions = {'NOUN', 'ADJF', 'ADJS', 'VERB', 'INFN'} + return pos in content_positions + except: + return False \ No newline at end of file diff --git a/app/keyword_extraction/paragraph_processor.py b/app/keyword_extraction/paragraph_processor.py new file mode 100644 index 0000000..db65006 --- /dev/null +++ b/app/keyword_extraction/paragraph_processor.py @@ -0,0 +1,104 @@ +""" +Paragraph processing utilities for splitting long texts into chunks. +""" + +import re +from typing import List, Dict + +class ParagraphProcessor: + """Handles processing of paragraphs into chunks.""" + + def __init__(self, max_chunk_size: int = 500, overlap: int = 50): + """ + Initialize paragraph processor. + + Args: + max_chunk_size: Maximum characters per chunk + overlap: Number of characters to overlap between chunks + """ + self.max_chunk_size = max_chunk_size + self.overlap = overlap + + def split_into_sentences(self, text: str, language: str = 'en') -> List[str]: + """Split text into sentences.""" + sentence_endings = r'[.!?]+[\s\n]+' + sentences = re.split(sentence_endings, text) + return [s.strip() for s in sentences if s.strip()] + + def split_paragraph_into_chunks(self, paragraph: str, language: str = 'en') -> List[Dict]: + """ + Split a paragraph into overlapping chunks. + + Returns: + List of dicts with 'chunk_id', 'text', 'start_sentence', 'end_sentence' + """ + sentences = self.split_into_sentences(paragraph, language) + + if not sentences: + return [] + + chunks = [] + current_chunk = [] + current_length = 0 + chunk_id = 1 + + for i, sentence in enumerate(sentences): + sentence_length = len(sentence) + + if current_length + sentence_length > self.max_chunk_size and current_chunk: + chunk_text = ' '.join(current_chunk) + chunks.append({ + 'chunk_id': chunk_id, + 'text': chunk_text, + 'start_sentence': chunk_id - 1, + 'end_sentence': i + }) + chunk_id += 1 + + # Start new chunk with overlap + overlap_sentences = [] + overlap_length = 0 + for s in reversed(current_chunk): + if overlap_length + len(s) <= self.overlap: + overlap_sentences.insert(0, s) + overlap_length += len(s) + else: + break + + current_chunk = overlap_sentences + current_length = overlap_length + + current_chunk.append(sentence) + current_length += sentence_length + 1 + + # Add the last chunk + if current_chunk: + chunk_text = ' '.join(current_chunk) + chunks.append({ + 'chunk_id': chunk_id, + 'text': chunk_text, + 'start_sentence': chunk_id - 1, + 'end_sentence': len(sentences) + }) + + return chunks + + def process_paragraph(self, paragraph: str, language: str = 'en') -> List[Dict]: + """ + Process a paragraph into chunks ready for keyword extraction. + """ + # Remove extra whitespace + paragraph = re.sub(r'\s+', ' ', paragraph).strip() + + # Check if paragraph needs to be split + if len(paragraph) <= self.max_chunk_size: + return [{ + 'chunk_id': 1, + 'text': paragraph, + 'is_full_paragraph': True + }] + + # Split into chunks + chunks = self.split_paragraph_into_chunks(paragraph, language) + + return chunks \ No newline at end of file diff --git a/app/keyword_extraction/requirements.txt b/app/keyword_extraction/requirements.txt new file mode 100644 index 0000000..489546f --- /dev/null +++ b/app/keyword_extraction/requirements.txt @@ -0,0 +1,5 @@ +sentence-transformers +keybert +numpy +nltk +pymorphy3 \ No newline at end of file diff --git a/app/keyword_extraction/stopwords.py b/app/keyword_extraction/stopwords.py new file mode 100644 index 0000000..011b5ff --- /dev/null +++ b/app/keyword_extraction/stopwords.py @@ -0,0 +1,128 @@ +""" +Language-specific stopwords for English and Russian. +""" + +STOPWORDS = { + 'en': { + # Предлоги + 'a', 'an', 'the', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', + 'through', 'across', 'along', 'around', 'behind', 'beneath', 'beside', + 'between', 'beyond', 'during', 'except', 'from', 'into', 'like', 'near', + 'off', 'onto', 'over', 'past', 'since', 'under', 'upon', 'within', 'without', + + # Союзы + 'and', 'or', 'but', 'so', 'for', 'nor', 'yet', 'because', 'if', 'then', + 'else', 'when', 'where', 'which', 'while', 'than', 'that', 'though', + 'although', 'whereas', 'whether', 'either', 'neither', 'both', + + # Местоимения + 'i', 'you', 'he', 'she', 'it', 'we', 'they', + 'me', 'him', 'her', 'us', 'them', + 'my', 'your', 'his', 'her', 'its', 'our', 'their', + 'mine', 'yours', 'hers', 'ours', 'theirs', + 'this', 'that', 'these', 'those', + 'what', 'which', 'who', 'whom', 'whose', + 'anyone', 'everyone', 'someone', + 'no one', 'nobody', + + # Частицы и модальные слова + 'not', 'no', 'never', 'just', 'even', 'only', 'very', 'so', 'too', 'also', + 'either', 'neither', 'both', 'all', 'some', 'any', 'no', 'every', 'each', + 'few', 'many', 'much', 'most', 'least', 'little', 'lot', 'lots', + + # Вспомогательные и модальные глаголы + 'be', 'am', 'are', 'is', 'was', 'were', 'been', 'being', 'have', 'has', + 'had', 'having', 'do', 'does', 'did', 'doing', 'can', 'could', 'will', + 'would', 'shall', 'should', 'may', 'might', 'must', 'ought', 'need', + + # Вопросительные слова + 'how', 'why', 'where', 'when', 'what', 'which', 'who', 'whom', 'whose', + + # Числительные + 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', + 'first', 'second', 'third', 'fourth', 'fifth', 'last', 'next', 'previous', + + # Короткие слова (обычно не несут смысла) + 'as', 'at', 'by', 'in', 'of', 'on', 'to', 'up', 'down', 'out', 'off', + 'over', 'under', 'again', 'further', 'once', 'here', 'there', 'now', 'then', + 'than', 'via', 'per', + }, + 'ru': { + # Предлоги + 'в', 'во', 'на', 'за', 'под', 'над', 'перед', 'после', 'через', 'сквозь', + 'между', 'среди', 'около', 'возле', 'мимо', 'вдоль', 'против', 'ради', + 'для', 'без', 'вместо', 'из', 'из-за', 'из-под', 'благодаря', 'согласно', + 'вопреки', 'наперекор', 'вследствие', 'ввиду', 'вроде', 'наподобие', + 'относительно', 'посредством', 'путём', 'через', 'сквозь', 'по', 'к', + 'у', 'о', 'об', 'обо', 'при', 'про', 'через', + + # Союзы + 'и', 'а', 'но', 'да', 'или', 'либо', 'то', 'если', 'же', 'ведь', 'однако', + 'зато', 'чтобы', 'потому', 'так', 'как', 'будто', 'словно', 'точно', 'чем', + 'нежели', 'хотя', 'пусть', 'пускай', 'лишь', 'только', 'также', 'тоже', + 'причём', 'притом', 'зачем', 'отчего', 'почему', 'потому', 'оттого', + + # Местоимения + 'я', 'ты', 'он', 'она', 'оно', 'мы', 'вы', 'они', 'меня', 'тебя', 'его', + 'ее', 'нас', 'вас', 'их', 'мне', 'тебе', 'ему', 'ей', 'нам', 'вам', 'им', + 'мой', 'твой', 'его', 'ее', 'наш', 'ваш', 'их', 'свой', 'этот', 'тот', + 'такой', 'таков', 'столько', 'сколько', 'несколько', 'весь', 'всякий', + 'каждый', 'любой', 'другой', 'иной', 'сам', 'самый', 'кто', 'что', + 'это', 'то', 'та', + + # Частицы + 'не', 'ни', 'бы', 'б', 'же', 'ж', 'ли', 'ль', 'пусть', 'пускай', 'да', + 'давай', 'ага', 'ой', 'ах', 'увы', 'вот', 'вон', 'разве', 'неужели', + 'едва', 'чуть', 'почти', 'прямо', 'точно', 'ровно', 'именно', + + # Вспомогательные и модальные глаголы + 'быть', 'стать', 'являться', 'становиться', 'бывать', 'бывал', 'бывали', 'бывало', 'будет', + 'будут', 'был', 'была', 'было', 'были', 'есть', 'суть', 'мочь', 'смочь', 'уметь', + 'хотеть', 'захотеть', 'желать', 'должен', 'должна', 'должно', 'должны', + 'обязан', 'годиться', 'пригодиться', 'следовать', 'полагаться', + + # Вопросительные слова + 'кто', 'некто', 'никто', 'что', 'нечто', 'ничто', 'какой', 'никакой', 'какая', + 'никакая', 'некая', 'какое', 'никакое', 'некое', 'какие', 'некие', 'никакие', + 'чей', 'чья', 'чьё', 'чьи', + 'который', 'некоторый', 'которая', 'некоторая', 'которое', 'некоторое', 'которые', 'некоторые', + 'где', 'нигде', 'куда', 'никуда', 'некуда', 'откуда', 'неоткуда', 'ниоткуда', + 'когда', 'некогда', 'почему', 'зачем', 'незачем', 'отчего', 'ниотчего', 'неотчего', + 'сколько', 'несколько', 'нисколько', 'насколько', 'нинасколько', + 'что', 'чего', 'чему', 'чем', 'чём', + 'то', 'того', 'тому', 'тем', 'том', + 'кто', 'кого', 'кому', 'кем', 'ком', + 'тот', 'того', 'тому', 'тем', + + # Числительные + 'один', 'одна', 'одно', 'одни', 'два', 'две', 'три', 'четыре', 'пять', + 'шесть', 'семь', 'восемь', 'девять', 'десять', + 'первый', 'второй', 'третий', 'четвёртый', 'пятый', + 'последний', 'предыдущий', 'следующий', + + # Короткие слова + 'из', 'со', 'об', 'обо', 'от', 'до', 'по', 'без', 'для', 'про', 'у', 'за', + 'над', 'под', 'перед', 'после', 'через', 'между', 'сквозь', 'вслед', + + # Междометия + 'о', 'ох', 'ах', 'эх', 'ух', 'ну', 'браво', 'ура', 'ого', 'ба', + } +} + +PUNCTUATION = { + 'en': r'[^\w\s]', + 'ru': r'[^\w\s\u0400-\u04FF]' # Preserve Cyrillic characters +} + +def get_stopwords(language: str = 'en'): + """Get stopwords for the specified language.""" + return STOPWORDS.get(language, STOPWORDS['en']) + +def get_punctuation_pattern(language: str = 'en'): + """Get punctuation pattern for the specified language.""" + return PUNCTUATION.get(language, PUNCTUATION['en']) + +# Дополнительная функция для проверки, является ли слово стоп-словом +def is_stopword(word: str, language: str = 'en') -> bool: + """Check if a word is a stopword.""" + return word.lower() in get_stopwords(language) \ No newline at end of file diff --git a/app/main.py b/app/main.py index aa3fc16..e001624 100644 --- a/app/main.py +++ b/app/main.py @@ -1,17 +1,99 @@ -import docs_parser - -# NOTE: все эти точно работают и работают хорошо -# (doc_p, _) = docs_parser.extract_text("parser/assets/text_and_tables.docx") -# (doc_p, _) = docs_parser.extract_text("parser/assets/text_and_tables.docx") -# (doc_p, _) = docs_parser.extract_text("parser/assets/some_text.docx") -# (doc_p, _) = docs_parser.extract_text("parser/assets/text_tables_png.docx") -# (doc_p, _) = docs_parser.extract_text("parser/assets/text_from_img.png") -# (doc_p, _) = docs_parser.extract_text("parser/assets/main.typ") -# (doc_p, _) = docs_parser.extract_text("parser/assets/main.pdf") -# (doc_p, _) = docs_parser.extract_text("parser/assets/too_many_png.docx") -# (doc_p, _) = docs_parser.extract_text("parser/assets/Presentation.pptx") -(doc_p, _) = docs_parser.extract_text("parser/assets/Book.xlsx") -print(doc_p) -# docs_parser.convert_to_new_format("parser/assets/old_docs.doc", "parser/assets/tests_results") -# docs_parser.convert_to_new_format("parser/assets/old_pres.ppt", "parser/assets/tests_results") -# docs_parser.convert_to_new_format("parser/assets/old_exel.xls", "parser/assets/tests_results") +""" +app/main.py - точка входа FastAPI-приложения. +""" +import os +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from sqlalchemy import text + +from app.database import engine +from app.embeddings import load_model, get_embedding_dimension + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +# ------------------------------------------ +# Lifespan: инициализация при старте +# ------------------------------------------ + +@asynccontextmanager +async def lifespan(app: FastAPI): + # 1. Проверяем подключение к БД + # Таблицами управляет Alembic - запустите `alembic upgrade head` перед стартом + try: + async with engine.connect() as conn: + await conn.execute(text("SELECT 1")) + logger.info("Database connection successful") + except Exception as e: + logger.error(f"Database connection failed: {e}") + + # 2. Загружаем модель эмбеддингов + load_model() + logger.info(f"Embedding model loaded, dimension: {get_embedding_dimension()}") + + yield + # (teardown при завершении - при необходимости добавить сюда) + + +# ------------------------------------------ +# Приложение +# ------------------------------------------ + +app = FastAPI( + title="Docs Search API", + description="Корпоративная RAG-система для работы с приватными документами", + version="0.1.0", + lifespan=lifespan, +) + +# CORS - разрешаем фронтенд на localhost +app.add_middleware( + CORSMiddleware, + allow_origins=[ + "http://localhost:3000", + "http://localhost:8000", + "http://127.0.0.1:3000", + "http://127.0.0.1:8000", + ], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# ------------------------------------------ +# Роутеры +# ------------------------------------------ + +from app.auth import router as auth_router # noqa: E402 +from app.documents import router as docs_router +from app.chat import router as chat_router +from app.groups import router as groups_router + +app.include_router(auth_router) +app.include_router(docs_router) +app.include_router(chat_router) +app.include_router(groups_router) + + +# ------------------------------------------ +# Системные эндпоинты +# ------------------------------------------ + +@app.get("/api", tags=["system"]) +def root(): + return {"status": "ok", "message": "Docs Search API is running"} + +app.mount("/ui", StaticFiles(directory="front-end", html=True), name="static") + + +@app.get("/health", tags=["system"]) +def health(): + db_url = os.getenv("DATABASE_URL", "not set") + # Скрываем пароль из URL для безопасного логирования + safe_url = db_url.split("@")[-1] if "@" in db_url else db_url + return {"status": "healthy", "database": safe_url} \ No newline at end of file diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..ceb9999 --- /dev/null +++ b/app/models.py @@ -0,0 +1,77 @@ +import uuid +from datetime import datetime +from sqlalchemy import Column, String, Integer, DateTime, ForeignKey, BigInteger, Index, Table +from sqlalchemy.dialects.postgresql import UUID, JSONB +from sqlalchemy.orm import relationship + +from app.database import Base + + +# ------------------------------------------ +# Ассоциативная таблица User ↔ Group +# ------------------------------------------ +user_groups = Table( + "user_groups", + Base.metadata, + Column("user_id", UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True), + Column("group_id", UUID(as_uuid=True), ForeignKey("groups.id", ondelete="CASCADE"), primary_key=True), +) + + +class User(Base): + __tablename__ = "users" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + username = Column(String, unique=True, nullable=False) + hashed_password = Column(String, nullable=False) + role = Column(String, default="user") + + groups = relationship("Group", secondary=user_groups, back_populates="members") + + +class Group(Base): + __tablename__ = "groups" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String, unique=True, nullable=False) + description = Column(String) + created_by = Column(UUID(as_uuid=True), ForeignKey("users.id")) + created_at = Column(DateTime, default=datetime.utcnow) + + members = relationship("User", secondary=user_groups, back_populates="groups") + + +class Document(Base): + __tablename__ = "documents" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + title = Column(String, nullable=False) + author = Column(String) + uploader_id = Column(UUID(as_uuid=True), ForeignKey("users.id")) + upload_date = Column(DateTime, default=datetime.utcnow) + last_edited = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + extension = Column(String) + size_bytes = Column(BigInteger) + description = Column(String) + file_path = Column(String) + is_available_to = Column(JSONB) # список ID пользователей, имеющих доступ + available_to_groups = Column(JSONB) # список ID групп, имеющих доступ + + +class Chunk(Base): + __tablename__ = "chunks" + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + document_id = Column(UUID(as_uuid=True), ForeignKey("documents.id", ondelete="CASCADE")) + chunk_index = Column(Integer) + text = Column(String) + keywords = Column(JSONB) + language = Column(String) + start_char = Column(Integer) + end_char = Column(Integer) + + +# Индексы для производительности +Index("idx_chunks_document_id", Chunk.document_id) +Index( + "idx_chunks_keywords_gin", + Chunk.keywords, + postgresql_using="gin", + postgresql_ops={"keywords": "jsonb_ops"}, +) \ No newline at end of file diff --git a/app/qdrant_client.py b/app/qdrant_client.py new file mode 100644 index 0000000..9589c44 --- /dev/null +++ b/app/qdrant_client.py @@ -0,0 +1,147 @@ +from qdrant_client import QdrantClient +from qdrant_client.http import models +from typing import List, Dict +import logging + +logger = logging.getLogger(__name__) + +# Подключение к Qdrant +qdrant = QdrantClient(host="localhost", port=6333) + +COLLECTION_NAME = "chunks" +VECTOR_SIZE = 312 + +def ensure_collection(): + """Создаёт коллекцию в Qdrant, если её нет""" + try: + collections = qdrant.get_collections().collections + if not any(c.name == COLLECTION_NAME for c in collections): + qdrant.create_collection( + collection_name=COLLECTION_NAME, + vectors_config=models.VectorParams( + size=VECTOR_SIZE, + distance=models.Distance.COSINE + ) + ) + logger.info(f"Collection '{COLLECTION_NAME}' created") + else: + logger.debug(f"Collection '{COLLECTION_NAME}' already exists") + except Exception as e: + logger.error(f"Failed to ensure collection: {e}") + +def index_chunk( + chunk_id: str, + document_id: str, + chunk_index: int, + text: str, + keywords: List[str], + language: str, + embedding: List[float] +) -> bool: + """Сохраняет вектор и метаданные чанка в Qdrant""" + ensure_collection() + try: + qdrant.upsert( + collection_name=COLLECTION_NAME, + points=[ + models.PointStruct( + id=chunk_id, + vector=embedding, + payload={ + "document_id": document_id, + "chunk_index": chunk_index, + "text": text, + "keywords": keywords, + "language": language + } + ) + ] + ) + logger.debug(f"Indexed chunk {chunk_id}") + return True + except Exception as e: + logger.error(f"Failed to index chunk {chunk_id}: {e}") + return False + +def semantic_search(query_embedding: List[float], top_k: int = 10) -> List[Dict]: + """Семантический поиск по вектору (KNN)""" + ensure_collection() + try: + # Используем query_points вместо search (для новых версий Qdrant) + results = qdrant.query_points( + collection_name=COLLECTION_NAME, + query=query_embedding, + limit=top_k, + with_payload=True + ) + return [ + { + "id": hit.id, + "score": hit.score, + **hit.payload + } + for hit in results.points + ] + except AttributeError: + # Для старых версий Qdrant (fallback) + try: + results = qdrant.search( + collection_name=COLLECTION_NAME, + query_vector=query_embedding, + limit=top_k, + with_payload=True + ) + return [ + { + "id": hit.id, + "score": hit.score, + **hit.payload + } + for hit in results + ] + except Exception as e2: + logger.error(f"Both search methods failed: {e2}") + return [] + except Exception as e: + logger.error(f"Semantic search failed: {e}") + return [] + +def delete_chunks_by_document(document_id: str) -> bool: + """Удаляет все чанки документа из Qdrant""" + try: + qdrant.delete( + collection_name=COLLECTION_NAME, + points_selector=models.FilterSelector( + filter=models.Filter( + must=[ + models.FieldCondition( + key="document_id", + match=models.MatchValue(value=document_id) + ) + ] + ) + ) + ) + logger.info(f"Deleted chunks for document {document_id}") + return True + except Exception as e: + logger.error(f"Failed to delete chunks for document {document_id}: {e}") + return False + +def delete_all_chunks(): + """Очищает всю коллекцию (для тестов)""" + try: + qdrant.delete_collection(COLLECTION_NAME) + logger.info("Collection deleted") + except Exception as e: + logger.error(f"Failed to delete collection: {e}") + +def get_chunk_count() -> int: + """Возвращает количество точек в коллекции""" + try: + ensure_collection() + info = qdrant.get_collection(COLLECTION_NAME) + return info.points_count + except Exception as e: + logger.error(f"Failed to get chunk count: {e}") + return 0 \ No newline at end of file diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 0000000..49c6818 --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,160 @@ +from pydantic import BaseModel +from uuid import UUID +from datetime import datetime + + +# ------------------------------------------ +# Устаревшие схемы (оставляем для совместимости) +# ------------------------------------------ + +class ParseResponse(BaseModel): + file_id: str + original_filename: str + parsed_file_path: str | None = None + status: str + + +# ------------------------------------------ +# Аутентификация +# ------------------------------------------ + +class UserRegisterRequest(BaseModel): + username: str + password: str + + +class TokenResponse(BaseModel): + """Возвращается при логине и обновлении токена.""" + access_token: str + refresh_token: str + token_type: str = "bearer" + + +class RefreshTokenRequest(BaseModel): + """Тело запроса POST /auth/refresh.""" + refresh_token: str + + +class UserResponse(BaseModel): + """Публичное представление пользователя.""" + id: UUID + username: str + role: str + + class Config: + from_attributes = True + + +# ------------------------------------------ +# Управление документами +# ------------------------------------------ + +class DocumentUploadResponse(BaseModel): + document_id: UUID + status: str + + +class DocumentResponse(BaseModel): + id: UUID + title: str + author: str | None = None + uploader_id: UUID | None = None + uploader_username: str | None = None + upload_date: datetime + last_edited: datetime + extension: str | None = None + size_bytes: int | None = None + description: str | None = None + is_available_to: list[str] | None = None + available_to_groups: list[str] | None = None + + class Config: + from_attributes = True + + +class PaginatedDocuments(BaseModel): + items: list[DocumentResponse] + total: int + page: int + pages: int + page_size: int + + +class DocumentUpdateRequest(BaseModel): + title: str | None = None + author: str | None = None + description: str | None = None + is_available_to: list[UUID] | None = None + available_to_groups: list[UUID] | None = None + + +class ChunkResponse(BaseModel): + id: UUID + document_id: UUID + chunk_index: int + text: str + keywords: list[str] | None = None + language: str | None = None + + class Config: + from_attributes = True + + +# ------------------------------------------ +# Поиск +# ------------------------------------------ + +class SearchRequest(BaseModel): + query: str + top_k: int = 10 + + +class SearchResultItem(BaseModel): + chunk_id: str + document_id: str + document_title: str + extension: str | None = None + text: str + keywords: list[str] | None = None + score: float + + +# ------------------------------------------ +# Чат (RAG) +# ------------------------------------------ + +class ChatRequest(BaseModel): + query: str + + +# ------------------------------------------ +# Группы и управление доступом +# ------------------------------------------ + +class GroupCreate(BaseModel): + name: str + description: str | None = None + + +class GroupMemberAdd(BaseModel): + user_ids: list[UUID] + + +class GroupResponse(BaseModel): + id: UUID + name: str + description: str | None = None + created_at: datetime + member_count: int | None = None + + class Config: + from_attributes = True + + +class UserSearchResult(BaseModel): + id: UUID + username: str + role: str + + class Config: + from_attributes = True diff --git a/app/search.py b/app/search.py new file mode 100644 index 0000000..1b64751 --- /dev/null +++ b/app/search.py @@ -0,0 +1,161 @@ +""" +Гибридный поиск: семантический (Qdrant) + по ключевым словам (PostgreSQL). +""" +import logging +from typing import List +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy import or_, and_, cast, String + +from app.models import User, Document, Chunk, user_groups +from app.qdrant_client import semantic_search +from app.embeddings import get_embedding +from app.schemas import SearchResultItem +from sqlalchemy.future import select + +logger = logging.getLogger(__name__) + + +def _get_accessible_doc_ids(documents: list[Document]) -> set[str]: + return {str(doc.id) for doc in documents} + + +async def hybrid_search( + query: str, + current_user: User, + db: AsyncSession, + top_k: int = 10, +) -> List[SearchResultItem]: + """ + Гибридный поиск: + 1. Получаем список документов, доступных пользователю. + 2. Семантический поиск в Qdrant по вектору запроса. + 3. Поиск по ключевым словам в PostgreSQL по чанкам этих документов. + 4. Объединяем, дедублицируем, сортируем по score и возвращаем top_k. + """ + + # -- 1. Список доступных документов ----------------------------------- + if current_user.role == "admin": + doc_result = await db.execute(select(Document)) + else: + # Получаем группы пользователя + gid_result = await db.execute( + select(user_groups.c.group_id).where(user_groups.c.user_id == current_user.id) + ) + user_group_ids = [str(row[0]) for row in gid_result.fetchall()] + + group_conditions = [ + Document.available_to_groups.has_key(gid) + for gid in user_group_ids + ] + + doc_query = select(Document).where( + or_( + Document.uploader_id == current_user.id, + and_( + or_(Document.is_available_to.is_(None), cast(Document.is_available_to, String).in_(('null', '[]'))), + or_(Document.available_to_groups.is_(None), cast(Document.available_to_groups, String).in_(('null', '[]'))) + ), + Document.is_available_to.has_key(str(current_user.id)), + *group_conditions, + ) + ) + doc_result = await db.execute(doc_query) + + accessible_docs = doc_result.scalars().all() + accessible_doc_ids = _get_accessible_doc_ids(accessible_docs) + doc_lookup = {str(doc.id): doc for doc in accessible_docs} + + if not accessible_doc_ids: + return [] + + # -- 2. Семантический поиск в Qdrant ---------------------------------- + query_embedding = get_embedding(query) + qdrant_results = semantic_search(query_embedding, top_k=top_k * 3) + + # Фильтруем по доступным документам + semantic_hits: dict[str, dict] = {} + for hit in qdrant_results: + doc_id = hit.get("document_id") + chunk_id = str(hit.get("id", "")) + if doc_id in accessible_doc_ids: + semantic_hits[chunk_id] = { + "chunk_id": chunk_id, + "document_id": doc_id, + "text": hit.get("text", ""), + "keywords": hit.get("keywords", []), + "score": float(hit.get("score", 0.0)), + } + + # -- 3. Поиск по ключевым словам в PostgreSQL ------------------------- + # Извлекаем слова из запроса (≥3 символа) для простого keyword-поиска + query_words = [w.lower() for w in query.split() if len(w) >= 3] + + keyword_hits: dict[str, dict] = {} + if query_words: + # Ищем чанки, у которых хотя бы одно ключевое слово содержит слово из запроса + chunk_query = select(Chunk).where( + Chunk.document_id.in_([ + # UUID-объекты для корректного сравнения + doc.id for doc in accessible_docs + ]) + ) + chunk_result = await db.execute(chunk_query) + all_chunks = chunk_result.scalars().all() + + for chunk in all_chunks: + chunk_kw_list = chunk.keywords or [] + chunk_kw_lower = [kw.lower() for kw in chunk_kw_list] + chunk_text_lower = (chunk.text or "").lower() + + # Считаем количество совпадений + matches = 0 + for word in query_words: + if any(word in kw for kw in chunk_kw_lower): + matches += 1 + elif word in chunk_text_lower: + matches += 0.5 # частичное совпадение в тексте + + if matches > 0: + kw_score = matches / len(query_words) # нормализуем в [0..1] + chunk_id = str(chunk.id) + keyword_hits[chunk_id] = { + "chunk_id": chunk_id, + "document_id": str(chunk.document_id), + "text": chunk.text or "", + "keywords": chunk.keywords or [], + "score": kw_score * 0.5, # масштабируем, чтобы не перекрыть семантику + } + + # -- 4. Объединение результатов ---------------------------------------- + merged: dict[str, dict] = {} + + for chunk_id, hit in semantic_hits.items(): + merged[chunk_id] = dict(hit) + + for chunk_id, hit in keyword_hits.items(): + if chunk_id in merged: + # Если чанк найден обоими методами - суммируем score + merged[chunk_id]["score"] += hit["score"] + else: + merged[chunk_id] = dict(hit) + + # -- 5. Обогащаем заголовком и расширением документа ------------------ + results: List[SearchResultItem] = [] + for hit in merged.values(): + doc = doc_lookup.get(hit["document_id"]) + if doc is None: + continue + results.append(SearchResultItem( + chunk_id=hit["chunk_id"], + document_id=hit["document_id"], + document_title=doc.title, + extension=doc.extension, + text=hit["text"], + keywords=hit["keywords"], + score=round(hit["score"], 4), + )) + + # Сортируем по убыванию релевантности и обрезаем до top_k + results.sort(key=lambda r: r.score, reverse=True) + return results[:top_k] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..bf1d7fd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +services: + postgres: + image: pgvector/pgvector:pg15 # используем готовый образ с pgvector (опционально, но можно оставить обычный postgres, если pgvector не нужен) + container_name: docs_postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: docs_db + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres -d docs_db" ] + interval: 5s + timeout: 5s + retries: 5 + networks: + - docs_network + + qdrant: + image: qdrant/qdrant:latest + container_name: docs_qdrant + ports: + - "6333:6333" # REST API + - "6334:6334" # gRPC API (опционально) + volumes: + - qdrant_storage:/qdrant/storage + networks: + - docs_network + # solr временно отключаем (можно закомментировать или удалить) + # solr: ... + +volumes: + postgres_data: + qdrant_storage: + +networks: + docs_network: + driver: bridge diff --git a/front-end/css/auth.css b/front-end/css/auth.css new file mode 100644 index 0000000..24ffa7a --- /dev/null +++ b/front-end/css/auth.css @@ -0,0 +1,35 @@ +.auth-container { + max-width: 400px; + margin: 4rem auto; + padding: 2rem; + text-align: center; +} + +.auth-container h2 { + margin-bottom: 1.5rem; + font-weight: 600; +} + +.input-group { + margin-bottom: 1rem; + text-align: left; +} + +.input-group label { + display: block; + margin-bottom: 0.5rem; + font-size: 0.875rem; + color: var(--text-muted); +} + +.auth-toggle { + margin-top: 1.5rem; + font-size: 0.875rem; + color: var(--text-muted); + cursor: pointer; + transition: color 0.2s; +} + +.auth-toggle:hover { + color: var(--primary); +} diff --git a/front-end/css/base.css b/front-end/css/base.css new file mode 100644 index 0000000..e88a446 --- /dev/null +++ b/front-end/css/base.css @@ -0,0 +1,54 @@ +* { + margin: 0; + padding: 0; + box-sizing: border-box; + font-family: 'Inter', sans-serif; +} + +body { + background: var(--bg-color); + color: var(--text-main); + min-height: 100vh; + display: flex; + flex-direction: column; + overflow-x: hidden; +} + +/* Glassmorphism utility */ +.glass { + background: var(--glass-bg); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + border: 1px solid var(--glass-border); + border-radius: 16px; +} + +/* Container */ +.container { + max-width: 1200px; + margin: 2rem auto; + padding: 0 1rem; + width: 100%; + flex: 1; +} + +/* Views */ +.view { + display: none; + animation: fadeIn 0.4s ease forwards; +} + +.view.active { + display: block; +} + +.chat-view.active { + display: flex; + flex-direction: column; + gap: 0; +} + +@keyframes fadeIn { + from { opacity: 0; transform: translateY(10px); } + to { opacity: 1; transform: translateY(0); } +} diff --git a/front-end/css/chat.css b/front-end/css/chat.css new file mode 100644 index 0000000..d5e60b4 --- /dev/null +++ b/front-end/css/chat.css @@ -0,0 +1,75 @@ +.chat-container { + display: flex; + flex-direction: column; + flex: 1; + min-height: 400px; + max-height: 600px; +} + +.chat-messages { + flex: 1; + overflow-y: auto; + padding: 1.5rem; + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +.chat-message { + display: flex; + flex-direction: column; + max-width: 85%; +} + +.chat-message.user { + align-self: flex-end; +} + +.chat-message.assistant { + align-self: flex-start; +} + +.message-content { + padding: 1rem 1.25rem; + border-radius: 12px; + line-height: 1.5; + font-size: 0.95rem; +} + +.chat-message.user .message-content { + background: var(--primary); + color: white; + border-bottom-right-radius: 4px; +} + +.chat-message.assistant .message-content { + background: var(--hover-bg); + color: var(--text-main); + border-bottom-left-radius: 4px; + border: 1px solid var(--glass-border); +} + +.chat-input-area { + display: flex; + gap: 0.75rem; + padding: 1rem; + border-top: 1px solid var(--glass-border); + background: var(--input-bg); + border-bottom-left-radius: 16px; + border-bottom-right-radius: 16px; +} + +/* Sources section inside assistant message */ +.sources-box { + margin-bottom: 0.75rem; + padding: 0.75rem; + background: var(--input-bg); + border-radius: 8px; + border: 1px dashed var(--glass-border); + font-size: 0.8rem; +} + +.sources-box .source-title { + color: var(--accent-text); + margin-right: 0.5rem; +} diff --git a/front-end/css/components.css b/front-end/css/components.css new file mode 100644 index 0000000..bd1f51a --- /dev/null +++ b/front-end/css/components.css @@ -0,0 +1,193 @@ +/* Buttons */ +.btn { + padding: 0.5rem 1rem; + border-radius: 8px; + font-weight: 500; + cursor: pointer; + transition: all 0.2s ease; + border: none; + outline: none; +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn-primary { + background: var(--primary); + color: white; +} + +.btn-primary:hover { + background: var(--primary-hover); + transform: translateY(-1px); +} + +.btn-outline { + background: transparent; + color: var(--text-main); + border: 1px solid var(--glass-border); +} + +.btn-outline:hover { + background: var(--hover-bg); +} + +.btn-danger { + background: var(--danger-bg); + color: var(--danger-text); + border: 1px solid var(--danger-border); +} + +.btn-danger:hover { + background: var(--danger-border); +} + +/* Badge */ +.badge { + display: inline-block; + padding: 0.25rem 0.5rem; + border-radius: 999px; + font-size: 0.75rem; + font-weight: 500; + background: var(--hover-bg); +} + +/* Toasts */ +#toast-container { + position: fixed; + bottom: 2rem; + right: 2rem; + display: flex; + flex-direction: column; + gap: 0.5rem; + z-index: 1000; +} + +.toast { + padding: 1rem 1.5rem; + border-radius: 8px; + font-weight: 500; + animation: slideIn 0.3s ease forwards; + box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.5); + display: flex; + align-items: center; + gap: 0.5rem; +} + +.toast.success { background: var(--success); color: #fff; } +.toast.error { background: var(--danger); color: #fff; } + +@keyframes slideIn { + from { transform: translateX(100%); opacity: 0; } + to { transform: translateX(0); opacity: 1; } +} + +/* Spinner */ +.spinner { + width: 20px; + height: 20px; + border: 2px solid rgba(255, 255, 255, 0.3); + border-radius: 50%; + border-top-color: white; + animation: spin 0.8s ease infinite; + display: inline-block; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +/* Input control (shared) */ +.input-control { + width: 100%; + padding: 0.75rem 1rem; + border-radius: 8px; + border: 1px solid var(--glass-border); + background: var(--input-bg); + color: var(--text-main); + font-size: 1rem; + transition: border-color 0.2s; +} + +.input-control:focus { + outline: none; + border-color: var(--primary); +} + +/* User chips */ +.user-chips { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; + margin-top: 0.4rem; +} + +.chip { + display: inline-flex; + align-items: center; + gap: 0.3rem; + background: var(--accent-bg); + border: 1px solid var(--accent-border); + color: var(--accent-text); + border-radius: 99px; + padding: 0.2rem 0.6rem; + font-size: 0.78rem; + font-weight: 500; +} + +.chip .chip-remove { + cursor: pointer; + opacity: 0.7; + font-size: 0.7rem; +} + +.chip .chip-remove:hover { opacity: 1; } + +/* Source link */ +.source-link { + cursor: pointer; + text-decoration: underline; + text-underline-offset: 2px; + transition: color 0.15s ease; +} + +.source-link:hover { + color: var(--accent-text); +} + +/* Dropdown for user search */ +.search-results-dropdown { + position: absolute; + top: 100%; + left: 0; + right: 0; + background: var(--panel-bg); + border: 1px solid var(--glass-border); + border-radius: 8px; + max-height: 180px; + overflow-y: auto; + z-index: 200; +} + +.search-results-dropdown .dropdown-item { + padding: 0.5rem 0.8rem; + cursor: pointer; + font-size: 0.85rem; + transition: background 0.15s; +} + +.search-results-dropdown .dropdown-item:hover { + background: var(--hover-bg); +} + +.search-wrap { + position: relative; + flex: 1; +} + +.user-search-row { + display: flex; + gap: 0.5rem; +} diff --git a/front-end/css/documents.css b/front-end/css/documents.css new file mode 100644 index 0000000..c8e6955 --- /dev/null +++ b/front-end/css/documents.css @@ -0,0 +1,86 @@ +/* Document filter bar */ +.docs-search-bar { + margin-bottom: 1.25rem; +} + +/* Docs grid */ +.docs-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + gap: 1.5rem; +} + +/* Document card */ +.doc-card { + padding: 1.5rem; + display: flex; + flex-direction: column; + gap: 1rem; + transition: transform 0.2s ease; +} + +.doc-card:hover { + transform: translateY(-4px); +} + +.doc-header { + display: flex; + justify-content: space-between; + align-items: flex-start; +} + +.doc-title { + font-weight: 600; + font-size: 1.1rem; + word-break: break-all; +} + +.doc-meta { + font-size: 0.8rem; + color: var(--text-muted); + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +/* Pagination */ +.docs-pagination { + display: flex; + justify-content: center; + align-items: center; + gap: 0.35rem; + margin-top: 2rem; + flex-wrap: wrap; +} + +.pagination-btn { + min-width: 2.25rem; + height: 2.25rem; + padding: 0 0.6rem; + border-radius: 8px; + border: 1px solid var(--glass-border); + background: var(--input-bg); + color: var(--text-main); + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: all 0.15s ease; +} + +.pagination-btn:hover:not(:disabled) { + background: var(--primary); + border-color: var(--primary); + color: white; +} + +.pagination-btn.active { + background: var(--primary); + border-color: var(--primary); + color: white; + font-weight: 600; +} + +.pagination-btn:disabled { + opacity: 0.35; + cursor: not-allowed; +} diff --git a/front-end/css/groups.css b/front-end/css/groups.css new file mode 100644 index 0000000..d83cf71 --- /dev/null +++ b/front-end/css/groups.css @@ -0,0 +1,85 @@ +.groups-list { + display: flex; + flex-direction: column; + gap: 1rem; + margin-top: 1rem; +} + +.group-card { + border-radius: 14px; + padding: 1.25rem 1.5rem; + transition: transform 0.2s; +} + +.group-card:hover { transform: translateY(-1px); } + +.group-card-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + cursor: pointer; +} + +.group-card-title { + font-size: 1rem; + font-weight: 600; +} + +.group-card-meta { + font-size: 0.8rem; + color: var(--text-muted); + margin-top: 0.2rem; +} + +.group-card-actions { + display: flex; + gap: 0.4rem; + flex-shrink: 0; +} + +/* Members panel (expandable) */ +.group-members-panel { + margin-top: 1rem; + padding-top: 1rem; + border-top: 1px solid var(--glass-border); + display: none; +} + +.group-members-panel.open { + display: block; +} + +.member-list { + display: flex; + flex-direction: column; + gap: 0.4rem; + margin-bottom: 0.75rem; +} + +.member-row { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.4rem 0.6rem; + border-radius: 8px; + background: var(--hover-bg); + font-size: 0.85rem; +} + +.member-row .member-info { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.member-role { + font-size: 0.7rem; + color: var(--text-muted); +} + +.add-member-row { + display: flex; + gap: 0.5rem; + margin-top: 0.5rem; +} diff --git a/front-end/css/layout.css b/front-end/css/layout.css new file mode 100644 index 0000000..883e664 --- /dev/null +++ b/front-end/css/layout.css @@ -0,0 +1,66 @@ +/* Header */ +header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem 2rem; + border-bottom: 1px solid var(--glass-border); + background: var(--header-bg); + backdrop-filter: blur(10px); + position: sticky; + top: 0; + z-index: 100; +} + +.logo { + font-size: 1.25rem; + font-weight: 700; + letter-spacing: -0.025em; + color: var(--accent-text); +} + +.nav-actions { + display: flex; + gap: 1rem; + align-items: center; +} + +/* Tab navigation */ +.tab-nav { + display: flex; + gap: 0.25rem; + background: var(--input-bg); + border: 1px solid var(--glass-border); + border-radius: 10px; + padding: 0.25rem; +} + +.tab-btn { + padding: 0.4rem 1rem; + border-radius: 8px; + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + border: none; + background: transparent; + color: var(--text-muted); + transition: all 0.2s ease; +} + +.tab-btn:hover { + color: var(--text-main); + background: var(--hover-bg); +} + +.tab-btn.active { + background: var(--primary); + color: white; +} + +/* Dashboard header bar */ +.dashboard-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} diff --git a/front-end/css/modals.css b/front-end/css/modals.css new file mode 100644 index 0000000..27d5080 --- /dev/null +++ b/front-end/css/modals.css @@ -0,0 +1,307 @@ +/* -- Base modal overlay --------------------------- */ +.doc-modal { + position: fixed; + inset: 0; + z-index: 500; + display: flex; + align-items: center; + justify-content: center; + padding: 1rem; + animation: fadeIn 0.2s ease forwards; +} + +.doc-modal-backdrop { + position: absolute; + inset: 0; + background: var(--backdrop-bg); + backdrop-filter: blur(4px); + cursor: pointer; +} + +/* -- Document Preview Panel ----------------------- */ +.doc-modal-panel { + position: relative; + z-index: 1; + width: 100%; + max-width: 960px; + height: 90vh; + display: flex; + flex-direction: column; + overflow: hidden; + animation: slideUp 0.25s ease forwards; +} + +@keyframes slideUp { + from { + transform: translateY(30px); + opacity: 0; + } + + to { + transform: translateY(0); + opacity: 1; + } +} + +.doc-modal-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 1rem 1.5rem; + border-bottom: 1px solid var(--glass-border); + flex-shrink: 0; + gap: 1rem; +} + +.doc-modal-title-wrap { + display: flex; + align-items: center; + gap: 0.6rem; + min-width: 0; +} + +.doc-modal-icon { + font-size: 1.25rem; + flex-shrink: 0; +} + +.doc-modal-title { + font-weight: 600; + font-size: 1rem; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.doc-modal-actions { + display: flex; + gap: 0.5rem; + flex-shrink: 0; +} + +.doc-modal-body { + flex: 1; + overflow: hidden; + display: flex; + flex-direction: column; +} + +.doc-modal-body iframe { + flex: 1; + width: 100%; + border: none; + background: var(--panel-bg); +} + +/* -- Download prompt (non-previewable) ------------ */ +.doc-download-prompt { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 1.5rem; + height: 100%; + padding: 2rem; + text-align: center; +} + +.doc-download-prompt .file-icon { + font-size: 5rem; + opacity: 0.8; +} + +.doc-download-prompt h3 { + font-size: 1.25rem; + font-weight: 600; +} + +.doc-download-prompt p { + color: var(--text-muted); + font-size: 0.9rem; + max-width: 400px; +} + +/* -- Document metadata grid ----------------------- */ +.doc-modal-meta { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + gap: 0.4rem 1.2rem; + background: var(--hover-bg); + border: 1px solid var(--glass-border); + border-radius: 0.5rem; + padding: 0.75rem 1rem; + font-size: 0.8rem; + color: var(--text-muted); + width: 100%; + max-width: 600px; + margin: 0 auto; +} + +.doc-modal-meta span strong { + color: var(--text-main); +} + +/* -- Upload / Access modal panel ------------------ */ +.upload-modal-panel { + background: var(--panel-bg); + border: 1px solid var(--glass-border); + position: relative; + z-index: 1; + width: 100%; + max-width: 580px; + max-height: 90vh; + display: flex; + flex-direction: column; + overflow: hidden; + animation: slideUp 0.25s ease forwards; +} + +.upload-modal-body { + flex: 1; + overflow-y: auto; + padding: 1.5rem; + display: flex; + flex-direction: column; + gap: 1.2rem; +} + +.upload-modal-footer { + padding: 1rem 1.5rem; + border-top: 1px solid var(--glass-border); + display: flex; + gap: 0.75rem; + justify-content: flex-end; + flex-shrink: 0; +} + +/* -- Form groups inside modals -------------------- */ +.form-group { + display: flex; + flex-direction: column; + gap: 0.4rem; +} + +.form-group label { + font-size: 0.8rem; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.form-group .input-control, +.form-group textarea { + background: var(--input-bg); + border: 1px solid var(--glass-border); + border-radius: 8px; + padding: 0.6rem 0.9rem; + color: var(--text-main); + font-family: inherit; + font-size: 0.9rem; + resize: vertical; + min-height: 70px; +} + +/* -- File drop zone ------------------------------- */ +.file-drop-zone { + border: 2px dashed var(--glass-border); + border-radius: 10px; + padding: 1.5rem; + text-align: center; + cursor: pointer; + transition: border-color 0.2s, background 0.2s; +} + +.file-drop-zone:hover, +.file-drop-zone.dragover { + border-color: var(--accent-text); + background: var(--hover-bg); +} + +.file-drop-zone .file-drop-icon { + font-size: 2.5rem; + display: block; + margin-bottom: 0.5rem; +} + +.file-drop-zone .file-selected-name { + color: var(--accent-text); + font-weight: 600; + font-size: 0.9rem; + margin-top: 0.3rem; + word-break: break-all; +} + +/* -- Access toggle buttons ------------------------ */ +.access-toggle { + display: flex; + gap: 0.5rem; + padding: 0.3rem; + background: var(--input-bg); + border-radius: 10px; + margin-bottom: 0.5rem; +} + +.access-toggle-btn { + flex: 1; + padding: 0.45rem 0.75rem; + border-radius: 8px; + font-size: 0.8rem; + font-weight: 500; + background: transparent; + color: var(--text-muted); + border: none; + cursor: pointer; + transition: all 0.2s; +} + +.access-toggle-btn.active { + background: var(--accent-bg); + color: var(--accent-text); + border: 1px solid var(--accent-border); +} + +.access-panel { + display: none; + flex-direction: column; + gap: 0.7rem; +} + +.access-panel.visible { + display: flex; +} + +/* -- Tag multi-select (group checkboxes) ---------- */ +.tag-select-wrap { + background: var(--hover-bg); + border: 1px solid var(--glass-border); + border-radius: 8px; + padding: 0.5rem; + max-height: 160px; + overflow-y: auto; +} + +.tag-item { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.4rem 0.6rem; + border-radius: 6px; + font-size: 0.85rem; + cursor: pointer; + transition: background 0.15s; +} + +.tag-item:hover { + background: var(--hover-bg); +} + +.tag-item input[type="checkbox"] { + margin-right: 0.5rem; + accent-color: var(--primary); +} + +.tag-item .member-count { + font-size: 0.75rem; + color: var(--text-muted); +} \ No newline at end of file diff --git a/front-end/css/search.css b/front-end/css/search.css new file mode 100644 index 0000000..21f9c59 --- /dev/null +++ b/front-end/css/search.css @@ -0,0 +1,114 @@ +/* Search bar */ +.search-bar { + display: flex; + gap: 0.75rem; + align-items: center; + padding: 0.75rem 1rem; + margin-bottom: 2rem; +} + +.search-input { + flex: 1; + background: transparent; + border: none; + outline: none; + color: var(--text-main); + font-size: 1rem; + font-family: 'Inter', sans-serif; +} + +.search-input::placeholder { + color: var(--text-muted); +} + +.search-btn { + white-space: nowrap; + flex-shrink: 0; +} + +/* Search state (loading / empty) */ +.search-state { + text-align: center; + padding: 2rem; + color: var(--text-muted); + font-size: 0.95rem; +} + +/* Search results */ +.search-results { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.result-card { + padding: 1.25rem 1.5rem; + display: flex; + flex-direction: column; + gap: 0.75rem; + animation: fadeIn 0.3s ease forwards; +} + +.result-header { + display: flex; + justify-content: space-between; + align-items: center; + gap: 1rem; +} + +.result-source { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; + color: var(--text-muted); +} + +.result-source .doc-name { + font-weight: 600; + color: var(--accent-text); +} + +.score-badge { + display: inline-flex; + align-items: center; + gap: 0.3rem; + padding: 0.2rem 0.6rem; + border-radius: 999px; + font-size: 0.7rem; + font-weight: 600; + background: var(--accent-bg); + border: 1px solid var(--accent-border); + color: var(--accent-text); + white-space: nowrap; +} + +.result-text { + font-size: 0.9rem; + line-height: 1.65; + color: var(--text-main); + opacity: 0.9; +} + +.result-text mark { + background: var(--accent-bg); + color: var(--text-main); + border-radius: 3px; + padding: 0 2px; +} + +.result-keywords { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; +} + +.kw-tag { + padding: 0.2rem 0.6rem; + border-radius: 999px; + font-size: 0.7rem; + font-weight: 500; + background: rgba(16, 185, 129, 0.1); + border: 1px solid rgba(16, 185, 129, 0.2); + color: #6ee7b7; +} diff --git a/front-end/css/variables.css b/front-end/css/variables.css new file mode 100644 index 0000000..3f8ade1 --- /dev/null +++ b/front-end/css/variables.css @@ -0,0 +1,51 @@ +:root { + /* Brand Colors (Common) */ + --primary: #6366f1; + --primary-hover: #4f46e5; + --danger: #ef4444; + --success: #10b981; + + /* Light Theme (Default) */ + --bg-color: #f8fafc; + --text-main: #0f172a; + --text-muted: #64748b; + --glass-bg: rgba(255, 255, 255, 0.7); + --glass-border: rgba(0, 0, 0, 0.1); + --panel-bg: #ffffff; + --hover-bg: rgba(0, 0, 0, 0.05); + --input-bg: rgba(0, 0, 0, 0.05); + --backdrop-bg: rgba(0, 0, 0, 0.3); + + /* Accents (Light) */ + --accent-text: #4f46e5; + --accent-bg: rgba(99, 102, 241, 0.1); + --accent-border: rgba(99, 102, 241, 0.2); + + --header-bg: rgba(248, 250, 252, 0.8); + --danger-bg: rgba(239, 68, 68, 0.1); + --danger-text: #dc2626; + --danger-border: rgba(239, 68, 68, 0.2); +} + +[data-theme="dark"] { + /* Dark Theme */ + --bg-color: #0f172a; + --text-main: #f8fafc; + --text-muted: #94a3b8; + --glass-bg: rgba(255, 255, 255, 0.05); + --glass-border: rgba(255, 255, 255, 0.1); + --panel-bg: #1e1e2e; + --hover-bg: rgba(255, 255, 255, 0.05); + --input-bg: rgba(0, 0, 0, 0.2); + --backdrop-bg: rgba(0, 0, 0, 0.7); + + /* Accents (Dark) */ + --accent-text: #818cf8; + --accent-bg: rgba(99, 102, 241, 0.15); + --accent-border: rgba(99, 102, 241, 0.3); + + --header-bg: rgba(15, 23, 42, 0.5); + --danger-bg: rgba(239, 68, 68, 0.2); + --danger-text: #fca5a5; + --danger-border: rgba(239, 68, 68, 0.3); +} diff --git a/front-end/index.html b/front-end/index.html new file mode 100644 index 0000000..7f8dca9 --- /dev/null +++ b/front-end/index.html @@ -0,0 +1,177 @@ + + + + + + Enterprise Docs Search + + + + + + + + + + + + + + + +
+ + +
+ +
+ +
+
+
+ +
+ +
+

Вход

+
+
+ + +
+
+ + +
+ +
+
Нет аккаунта? Зарегистрируйтесь
+
+ + +
+
+

Ваши документы

+ +
+ +
+ +
+ +
+ +
+ +
+
+ + +
+
+

Поиск по документам

+
+ + + + +
+
+ + +
+
+

Управление группами

+ +
+
+ +
+
+ + +
+
+

ИИ-Ассистент

+
+ +
+
+
+
Здравствуйте! Я ваш ИИ-ассистент. Задайте мне вопрос по загруженным корпоративным документам.
+
+
+ +
+ + +
+
+
+
+ +
+ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/front-end/js/auth.js b/front-end/js/auth.js new file mode 100644 index 0000000..516fd78 --- /dev/null +++ b/front-end/js/auth.js @@ -0,0 +1,64 @@ +// ------------------------------------------------------------- +// Auth Functions +// ------------------------------------------------------------- +async function refreshToken() { + const refresh = localStorage.getItem('refresh_token'); + if (!refresh) return false; + try { + const res = await fetch(`${API_URL}/auth/refresh`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ refresh_token: refresh }), + }); + if (res.ok) { + const data = await res.json(); + localStorage.setItem('access_token', data.access_token); + localStorage.setItem('refresh_token', data.refresh_token); + return true; + } + } catch (e) { } + return false; +} + +function logout() { + localStorage.removeItem('access_token'); + localStorage.removeItem('refresh_token'); + state.isAuthenticated = false; + state.user = null; + state.activeTab = 'dashboard'; + state.documents = []; + state.groups = []; + state.docsPagination = { total: 0, page: 1, pages: 1, page_size: 12 }; + + if (typeof docsCurrentPage !== 'undefined') docsCurrentPage = 1; + if (typeof docsFilterQuery !== 'undefined') docsFilterQuery = ''; + + // Сброс UI-полей + const searchInput = document.getElementById('search-input'); + if (searchInput) searchInput.value = ''; + + const searchResults = document.getElementById('search-results'); + if (searchResults) searchResults.innerHTML = ''; + + const chatInput = document.getElementById('chat-input'); + if (chatInput) chatInput.value = ''; + + const chatMessages = document.getElementById('chat-messages'); + if (chatMessages) chatMessages.innerHTML = ''; + + if (typeof render === 'function') render(); +} + +async function checkAuth() { + if (localStorage.getItem('access_token')) { + try { + state.user = await apiFetch('/auth/me'); + state.isAuthenticated = true; + } catch (e) { + logout(); + } + } + if (typeof render === 'function') render(); +} + +window.logout = logout; diff --git a/front-end/js/chat.js b/front-end/js/chat.js new file mode 100644 index 0000000..c74316b --- /dev/null +++ b/front-end/js/chat.js @@ -0,0 +1,122 @@ +// ------------------------------------------------------------- +// Chat Functions +// ------------------------------------------------------------- +function appendMessage(role, content) { + const messagesContainer = document.getElementById('chat-messages'); + const msgDiv = document.createElement('div'); + msgDiv.className = `chat-message ${role}`; + + const contentDiv = document.createElement('div'); + contentDiv.className = 'message-content'; + // Using innerHTML to allow basic formatting and sources + contentDiv.innerHTML = content.replace(/\n/g, '
'); + + msgDiv.appendChild(contentDiv); + messagesContainer.appendChild(msgDiv); + messagesContainer.scrollTop = messagesContainer.scrollHeight; + + return contentDiv; +} + +async function performChat(query) { + appendMessage('user', query); + + const submitBtn = document.getElementById('chat-submit-btn'); + const inputField = document.getElementById('chat-input'); + + submitBtn.disabled = true; + inputField.disabled = true; + + // Add a placeholder message for the assistant + const assistantMsgContent = appendMessage('assistant', ' Думаю...'); + + try { + const token = localStorage.getItem('access_token'); + const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }; + + const response = await fetch(`${API_URL}/chat`, { + method: 'POST', + headers: headers, + body: JSON.stringify({ query: query }) + }); + + if (response.status === 401) { + const refreshed = await refreshToken(); + if (!refreshed) { + logout(); + throw new Error('Session expired'); + } + // Retry once + headers['Authorization'] = `Bearer ${localStorage.getItem('access_token')}`; + // For simplicity, we just throw here and let the user click again + // but normally we would await fetch again. + } + + if (!response.ok) { + throw new Error(`API Error: ${response.status}`); + } + + // Streaming logic + const reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8"); + + let fullText = ""; + let sourcesHtml = ""; + assistantMsgContent.innerHTML = ""; // clear spinner + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + const chunkText = decoder.decode(value, { stream: true }); + const lines = chunkText.split('\n').filter(l => l.trim() !== ''); + + for (const line of lines) { + try { + const data = JSON.parse(line); + + if (data.type === 'sources') { + if (data.sources && data.sources.length > 0) { + const sourceLinks = data.sources.map(s => { + const ext = (s.extension || '').replace('.', '').toLowerCase(); + return ` + ${EXT_ICONS[ext] || ''} ${escapeHtml(s.document_title)} + `; + }).join(' '); + sourcesHtml = `
+ Источники:
+ ${sourceLinks} +
`; + assistantMsgContent.innerHTML = sourcesHtml + fullText; + } + } else if (data.type === 'content') { + fullText += escapeHtml(data.content); + assistantMsgContent.innerHTML = sourcesHtml + fullText.replace(/\n/g, '
'); + document.getElementById('chat-messages').scrollTop = document.getElementById('chat-messages').scrollHeight; + } else if (data.type === 'error') { + fullText += `
[Ошибка: ${escapeHtml(data.content)}]`; + assistantMsgContent.innerHTML = sourcesHtml + fullText.replace(/\n/g, '
'); + } + } catch (e) { + console.error("Parse error chunk:", line, e); + } + } + } + + } catch (e) { + assistantMsgContent.innerHTML = `Ошибка: ${e.message}`; + } finally { + submitBtn.disabled = false; + inputField.disabled = false; + inputField.focus(); + } +} +window.performChat = performChat; diff --git a/front-end/js/doc-access.js b/front-end/js/doc-access.js new file mode 100644 index 0000000..28e86d6 --- /dev/null +++ b/front-end/js/doc-access.js @@ -0,0 +1,106 @@ +// ------------------------------------------------------------- +// Document Access (Groups) Modal +// ------------------------------------------------------------- + +let docAccessDocId = null; +let docAccessSelectedGroups = new Set(); + +window.openDocAccessModal = async function (docId, currentGroupsRaw) { + docAccessDocId = docId; + docAccessSelectedGroups = new Set( + Array.isArray(currentGroupsRaw) ? currentGroupsRaw : [] + ); + + // Получаем актуальные группы пользователя (уже в state.groups после fetchGroups) + if (!state.groups.length) await fetchGroups(false); + + const modal = document.getElementById('group-modal'); + + // Все группы, которые пользователь может назначить + // Админ видит все (но пока используем только те, что в state.groups) + const availableGroups = state.groups; + + modal.innerHTML = ` +
+
+

Доступ к документу

+ +
+
+

+ Выберите группы, которые имеют доступ к документу.
+ Если не выбрать ни одной - документ станет публичным. +

+ + + ${availableGroups.length === 0 ? ` +
+ Вы не состоите ни в одной группе. Документ останется публичным. +
+ ` : ` +
+ ${availableGroups.map(g => ` + + `).join('')} +
+ `} + +
+ ${docAccessSelectedGroups.size === 0 ? 'Документ публичный' : `Доступ ограничен: ${docAccessSelectedGroups.size} гр.`} +
+
+
+ + +
+
+ `; + + modal.style.display = 'flex'; + document.body.style.overflow = 'hidden'; +} + +window.toggleDocAccessGroup = function (groupId, isChecked) { + if (isChecked) docAccessSelectedGroups.add(groupId); + else docAccessSelectedGroups.delete(groupId); + + const statusEl = document.getElementById('doc-access-status'); + if (statusEl) { + statusEl.textContent = docAccessSelectedGroups.size === 0 + ? 'Документ публичный' + : `Доступ ограничен: ${docAccessSelectedGroups.size} гр.`; + } +} + +window.closeDocAccessModal = function () { + document.getElementById('group-modal').style.display = 'none'; + document.body.style.overflow = ''; + docAccessDocId = null; + docAccessSelectedGroups.clear(); +} + +window.submitDocAccess = async function () { + if (!docAccessDocId) return; + try { + await apiFetch(`/documents/${docAccessDocId}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + available_to_groups: Array.from(docAccessSelectedGroups), + }), + }); + showToast('Доступ обновлён'); + closeDocAccessModal(); + fetchDocuments(); + } catch (e) { + showToast('Ошибка при обновлении доступа', 'error'); + console.error(e); + } +} diff --git a/front-end/js/documents.js b/front-end/js/documents.js new file mode 100644 index 0000000..b523bb5 --- /dev/null +++ b/front-end/js/documents.js @@ -0,0 +1,487 @@ +// ------------------------------------------------------------- +// Document Functions +// ------------------------------------------------------------- +const DOCS_PAGE_SIZE = 12; +let docsCurrentPage = 1; +let docsFilterQuery = ''; + +async function fetchDocuments() { + try { + const params = new URLSearchParams({ + search: docsFilterQuery, + page: docsCurrentPage, + page_size: DOCS_PAGE_SIZE, + }); + const data = await apiFetch(`/documents?${params}`); + state.documents = data.items; + state.docsPagination = { total: data.total, page: data.page, pages: data.pages, page_size: data.page_size }; + renderDocuments(); + } catch (e) { console.error(e); } +} + +async function deleteDocument(id) { + if (!confirm('Вы уверены, что хотите удалить этот документ?')) return; + try { + await apiFetch(`/documents/${id}`, { method: 'DELETE' }); + showToast('Документ удален'); + fetchDocuments(); + } catch (e) { console.error(e); } +} + +window.deleteDocument = deleteDocument; + +function onDocsFilter(value) { + docsFilterQuery = value.trim().toLowerCase(); + docsCurrentPage = 1; + fetchDocuments(); +} +window.onDocsFilter = onDocsFilter; + +const INLINE_EXTS = new Set(['pdf', 'txt', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg']); + +const EXT_ICONS = { + pdf: 'PDF', txt: 'TXT', png: 'IMG', jpg: 'IMG', jpeg: 'IMG', gif: 'IMG', + webp: 'IMG', svg: 'IMG', docx: 'DOC', doc: 'DOC', xlsx: 'XLS', xls: 'XLS', + pptx: 'PPT', ppt: 'PPT', +}; + +function renderDocuments() { + const grid = document.getElementById('docs-grid'); + const paginationEl = document.getElementById('docs-pagination'); + + if (!state.documents || state.documents.length === 0) { + grid.innerHTML = `

${docsFilterQuery ? 'По вашему запросу документы не найдены.' : 'Документы не найдены.'}

`; + if (paginationEl) paginationEl.innerHTML = ''; + return; + } + + grid.innerHTML = state.documents.map(doc => { + const extRaw = (doc.extension || '').replace('.', '').toLowerCase(); + const extLabel = extRaw ? extRaw.toUpperCase() : 'UNKNOWN'; + const docIcon = EXT_ICONS[extRaw] || ''; + const uploaderLabel = doc.uploader_username ? `Загрузил: ${escapeHtml(doc.uploader_username)}` : ''; + return ` +
+
+
${docIcon} ${escapeHtml(doc.title || 'Untitled')}
+ ${extLabel} +
+
+ Размер: ${formatBytes(doc.size_bytes)} + Дата: ${new Date(doc.upload_date).toLocaleDateString()} + ${uploaderLabel ? `${uploaderLabel}` : ''} +
+
+ + ${(state.user.role === 'admin' || state.user.id === doc.uploader_id) ? ` + + + ` : ''} +
+
`; + }).join(''); + + // Пагинация по метаданным с сервера + if (paginationEl) { + const { page, pages } = state.docsPagination || { page: 1, pages: 1 }; + if (pages <= 1) { + paginationEl.innerHTML = ''; + } else { + const btns = []; + for (let i = 1; i <= pages; i++) { + btns.push(``); + } + paginationEl.innerHTML = ` + + ${btns.join('')} + + `; + } + } +} + +window.goToDocsPage = function (page) { + const { pages } = state.docsPagination || { pages: 1 }; + if (page < 1 || page > pages) return; + docsCurrentPage = page; + fetchDocuments(); + document.getElementById('docs-grid').scrollIntoView({ behavior: 'smooth', block: 'start' }); +} + +// ------------------------------------------------------------- +// Document Preview Modal +// ------------------------------------------------------------- +async function openDocModal(documentId, documentTitle, extension) { + const modal = document.getElementById('doc-modal'); + const icon = document.getElementById('modal-icon'); + const title = document.getElementById('modal-title'); + const body = document.getElementById('modal-body'); + const downloadBtn = document.getElementById('modal-download-btn'); + + const ext = (extension || '').replace('.', '').toLowerCase(); + + // Показываем модалку сразу с лоадером + icon.textContent = EXT_ICONS[ext] || ''; + title.textContent = documentTitle || 'Document'; + body.innerHTML = `
+ Загрузка... +
`; + downloadBtn.href = '#'; + downloadBtn.removeAttribute('download'); + modal.style.display = 'flex'; + document.body.style.overflow = 'hidden'; + + // Загружаем метаданные документа + let docMeta = null; + try { + const token = localStorage.getItem('access_token'); + const metaRes = await fetch(`${API_URL}/documents/${documentId}`, { + headers: { 'Authorization': `Bearer ${token}` } + }); + if (metaRes.ok) docMeta = await metaRes.json(); + } catch (_) { } + + const metaHtml = docMeta ? ` +
+ ${docMeta.author ? `Автор: ${escapeHtml(docMeta.author)}` : ''} + ${docMeta.uploader_username ? `Загрузил: ${escapeHtml(docMeta.uploader_username)}` : ''} + ${docMeta.upload_date ? `Дата: ${new Date(docMeta.upload_date).toLocaleDateString('ru-RU')}` : ''} + ${docMeta.size_bytes ? `Размер: ${formatBytes(docMeta.size_bytes)}` : ''} + ${docMeta.description ? `${escapeHtml(docMeta.description)}` : ''} +
` : ''; + + // Загружаем файл через fetch с авторизацией + try { + const token = localStorage.getItem('access_token'); + const response = await fetch(`${API_URL}/documents/${documentId}/content`, { + headers: { 'Authorization': `Bearer ${token}` } + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + + // Ссылка на скачивание + downloadBtn.href = blobUrl; + downloadBtn.download = documentTitle || `document.${ext}`; + + // Тело модалки + if (INLINE_EXTS.has(ext)) { + body.innerHTML = ` + ${metaHtml} + `; + } else { + body.innerHTML = ` +
+
${EXT_ICONS[ext] || 'DOC'}
+

${escapeHtml(documentTitle || 'Document')}

+ ${metaHtml} +

Формат .${ext.toUpperCase()} нельзя отобразить прямо в браузере. + Нажмите кнопку ниже, чтобы скачать файл.

+ + Скачать файл + +
`; + } + + // Сохраняем blobUrl для освобождения при закрытии + modal.dataset.blobUrl = blobUrl; + + } catch (e) { + body.innerHTML = `
+ Ошибка загрузки файла: ${escapeHtml(e.message)} +
`; + } +} + +function closeDocModal() { + const modal = document.getElementById('doc-modal'); + // Освобождаем Blob URL чтобы не было утечки памяти + if (modal.dataset.blobUrl) { + URL.revokeObjectURL(modal.dataset.blobUrl); + delete modal.dataset.blobUrl; + } + modal.style.display = 'none'; + document.getElementById('modal-body').innerHTML = ''; + document.body.style.overflow = ''; +} +window.openDocModal = openDocModal; +window.closeDocModal = closeDocModal; + +// ------------------------------------------------------------- +// Upload Modal & Logic +// ------------------------------------------------------------- +let pendingUploadFile = null; +let selectedUploadGroups = new Set(); +let selectedUploadUsers = new Set(); + +const openUploadBtn = document.getElementById('open-upload-modal-btn'); +if (openUploadBtn) { + openUploadBtn.addEventListener('click', () => openUploadModal()); +} + +async function openUploadModal(file = null) { + pendingUploadFile = file; + selectedUploadGroups.clear(); + selectedUploadUsers.clear(); + + const modal = document.getElementById('upload-modal'); + + // Fetch users/groups for dropdowns + if (state.user) { + if (!state.groups.length && typeof fetchGroups === 'function') await fetchGroups(false); // don't render view, just fetch + } + + modal.innerHTML = ` +
+
+

Загрузка документа

+ +
+
+ + + + + +
+ + +
+
+ + +
+ + +
+ +
+ + +
+ +
+
Выберите группы или пользователей, которым будет доступен этот документ.
+ + +
+ ${state.groups.map(g => ` + + `).join('')} + ${state.groups.length === 0 ? '
Нет доступных групп
' : ''} +
+ + ${state.user && state.user.role === 'admin' ? ` + +
+ + +
+
+ ` : ''} +
+
+ +
+
+ + +
+
+ `; + + modal.style.display = 'flex'; + document.body.style.overflow = 'hidden'; + + // File input handlers + const dropZone = document.getElementById('modal-drop-zone'); + const fileIn = document.getElementById('modalFileInput'); + + dropZone.addEventListener('dragover', (e) => { e.preventDefault(); dropZone.classList.add('dragover'); }); + dropZone.addEventListener('dragleave', () => dropZone.classList.remove('dragover')); + dropZone.addEventListener('drop', (e) => { + e.preventDefault(); + dropZone.classList.remove('dragover'); + if (e.dataTransfer.files.length) { + pendingUploadFile = e.dataTransfer.files[0]; + document.getElementById('modal-drop-text').textContent = 'Файл выбран'; + document.getElementById('modal-file-name').textContent = pendingUploadFile.name; + } + }); + fileIn.addEventListener('change', (e) => { + if (e.target.files.length) { + pendingUploadFile = e.target.files[0]; + document.getElementById('modal-drop-text').textContent = 'Файл выбран'; + document.getElementById('modal-file-name').textContent = pendingUploadFile.name; + } + }); +} + +function closeUploadModal() { + document.getElementById('upload-modal').style.display = 'none'; + document.body.style.overflow = ''; + pendingUploadFile = null; +} +window.closeUploadModal = closeUploadModal; + +window.toggleAccessType = function (type) { + const btnAll = document.getElementById('btn-access-all'); + const btnRestricted = document.getElementById('btn-access-restricted'); + const panel = document.getElementById('restricted-panel'); + const uploadBtn = document.querySelector('.upload-modal-footer .btn-primary'); + + const userHasNoGroups = state.user && state.user.role !== 'admin' && state.groups.length === 0; + + if (type === 'all') { + btnAll.classList.add('active'); + btnRestricted.classList.remove('active'); + panel.classList.remove('visible'); + selectedUploadGroups.clear(); + selectedUploadUsers.clear(); + document.querySelectorAll('#upload-group-list input[type="checkbox"]').forEach(cb => cb.checked = false); + renderUploadSelectedUsers(); + if (uploadBtn) { + uploadBtn.disabled = false; + uploadBtn.title = ''; + } + } else { + btnRestricted.classList.add('active'); + btnAll.classList.remove('active'); + panel.classList.add('visible'); + if (uploadBtn && userHasNoGroups) { + uploadBtn.disabled = true; + uploadBtn.title = 'Вы не состоите ни в одной группе - невозможно ограничить доступ'; + } + } +} + +window.toggleUploadGroup = function (groupId, isChecked) { + if (isChecked) selectedUploadGroups.add(groupId); + else selectedUploadGroups.delete(groupId); +} + +// User Search for modal +let userSearchTimeout = null; +window.debounceUserSearch = function (query, context) { + clearTimeout(userSearchTimeout); + userSearchTimeout = setTimeout(() => searchUsersApi(query, context), 300); +} + +async function searchUsersApi(query, context) { + const dropdown = document.getElementById(context === 'upload' ? 'upload-user-results' : 'group-user-results'); + if (!query || query.length < 2) { + dropdown.style.display = 'none'; + return; + } + + try { + const users = await apiFetch(`/auth/users?q=${encodeURIComponent(query)}`); + if (users.length === 0) { + dropdown.innerHTML = '
Пользователи не найдены
'; + } else { + dropdown.innerHTML = users.map(u => ` +
+ ${escapeHtml(u.username)} (${u.role}) +
+ `).join(''); + } + dropdown.style.display = 'block'; + } catch (e) { + console.error("User search failed", e); + } +} + +window.selectUser = function (id, username, context) { + document.getElementById(context === 'upload' ? 'upload-user-results' : 'group-user-results').style.display = 'none'; + document.getElementById(context === 'upload' ? 'upload-user-search' : 'group-user-search').value = ''; + + if (context === 'upload') { + selectedUploadUsers.add({ id, username }); + renderUploadSelectedUsers(); + } else { + selectedGroupUsers.add({ id, username }); + if (typeof renderGroupSelectedUsers === 'function') renderGroupSelectedUsers(); + } +} + +window.removeUploadUser = function (id) { + for (let u of selectedUploadUsers) { + if (u.id === id) { selectedUploadUsers.delete(u); break; } + } + renderUploadSelectedUsers(); +} + +function renderUploadSelectedUsers() { + const container = document.getElementById('upload-selected-users'); + if (!container) return; + container.innerHTML = Array.from(selectedUploadUsers).map(u => ` +
+ ${escapeHtml(u.username)} + ✕ +
+ `).join(''); +} + +async function submitUpload() { + if (!pendingUploadFile) { + showToast('Пожалуйста, сначала выберите файл', 'error'); + return; + } + + const title = document.getElementById('upload-title').value.trim(); + const desc = document.getElementById('upload-desc').value.trim(); + + const formData = new FormData(); + formData.append('file', pendingUploadFile); + if (title) formData.append('title', title); + if (desc) formData.append('description', desc); + + const isRestricted = document.getElementById('btn-access-restricted').classList.contains('active'); + if (isRestricted) { + if (selectedUploadUsers.size > 0) { + formData.append('is_available_to', Array.from(selectedUploadUsers).map(u => u.id).join(',')); + } + if (selectedUploadGroups.size > 0) { + formData.append('available_to_groups', Array.from(selectedUploadGroups).join(',')); + } + } + + showToast('Загрузка...', 'success'); + closeUploadModal(); + + try { + await apiFetch('/documents/upload', { + method: 'POST', + body: formData, + }); + showToast('Файл успешно загружен!'); + if (state.activeTab === 'dashboard') fetchDocuments(); + } catch (e) { + console.error(e); + } +} +window.submitUpload = submitUpload; diff --git a/front-end/js/groups.js b/front-end/js/groups.js new file mode 100644 index 0000000..505bad1 --- /dev/null +++ b/front-end/js/groups.js @@ -0,0 +1,182 @@ +// ------------------------------------------------------------- +// Groups Management +// ------------------------------------------------------------- + +async function fetchGroups(render = true) { + try { + state.groups = await apiFetch('/groups'); + if (render) renderGroups(); + } catch (e) { console.error(e); } +} +window.fetchGroups = fetchGroups; + +function renderGroups() { + const container = document.getElementById('groups-list'); + if (!container) return; + + if (state.groups.length === 0) { + container.innerHTML = '

Группы не найдены.

'; + return; + } + + container.innerHTML = state.groups.map(g => ` +
+
+
+
${escapeHtml(g.name)}
+
${escapeHtml(g.description || 'Нет описания')} • Участников: ${g.member_count}
+
+
+ +
+
+
+ Загрузка участников... +
+
+ `).join(''); +} +window.renderGroups = renderGroups; + +window.deleteGroup = async function (id) { + if (!confirm('Удалить эту группу?')) return; + try { + await apiFetch(`/groups/${id}`, { method: 'DELETE' }); + showToast('Группа удалена'); + fetchGroups(); + } catch (e) { console.error(e); } +} + +window.toggleGroupMembers = async function (groupId) { + const panel = document.getElementById(`group-panel-${groupId}`); + if (panel.classList.contains('open')) { + panel.classList.remove('open'); + return; + } + + // Close others + document.querySelectorAll('.group-members-panel').forEach(p => p.classList.remove('open')); + panel.classList.add('open'); + + try { + const members = await apiFetch(`/groups/${groupId}/members`); + panel.innerHTML = ` +
+ ${members.length === 0 ? '
Пока нет участников
' : ''} + ${members.map(m => ` +
+
+ ${escapeHtml(m.username)} + ${m.role} +
+ +
+ `).join('')} +
+
+
+ + +
+
+ `; + } catch (e) { + panel.innerHTML = '
Не удалось загрузить участников
'; + } +} + +let addMemberTimeout = null; +window.debounceAddMemberSearch = function (query, groupId) { + clearTimeout(addMemberTimeout); + addMemberTimeout = setTimeout(() => searchAddMemberApi(query, groupId), 300); +} + +async function searchAddMemberApi(query, groupId) { + const dropdown = document.getElementById(`results-add-${groupId}`); + if (!query || query.length < 2) { + dropdown.style.display = 'none'; + return; + } + try { + const users = await apiFetch(`/auth/users?q=${encodeURIComponent(query)}`); + if (users.length === 0) { + dropdown.innerHTML = '
Пользователи не найдены
'; + } else { + dropdown.innerHTML = users.map(u => ` +
+ ${escapeHtml(u.username)} (${u.role}) +
+ `).join(''); + } + dropdown.style.display = 'block'; + } catch (e) { } +} + +window.addMemberToGroup = async function (groupId, userId) { + try { + await apiFetch(`/groups/${groupId}/members`, { + method: 'POST', + body: JSON.stringify({ user_ids: [userId] }) + }); + showToast('Пользователь добавлен'); + toggleGroupMembers(groupId); // Refresh open panel + toggleGroupMembers(groupId); + fetchGroups(true); // update member counts + } catch (e) { console.error(e); } +} + +window.removeMember = async function (groupId, userId) { + try { + await apiFetch(`/groups/${groupId}/members/${userId}`, { method: 'DELETE' }); + showToast('Пользователь исключен'); + toggleGroupMembers(groupId); // Refresh open panel + toggleGroupMembers(groupId); + fetchGroups(true); + } catch (e) { console.error(e); } +} + +// Create Group logic +window.openCreateGroupModal = function () { + const modal = document.getElementById('group-modal'); + modal.innerHTML = ` +
+
+

Создать группу

+ +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+ `; + modal.style.display = 'flex'; +} + +window.submitCreateGroup = async function () { + const name = document.getElementById('new-group-name').value.trim(); + const desc = document.getElementById('new-group-desc').value.trim(); + if (!name) return showToast('Название обязательно', 'error'); + + try { + await apiFetch('/groups', { + method: 'POST', + body: JSON.stringify({ name, description: desc }) + }); + showToast('Группа создана'); + document.getElementById('group-modal').style.display = 'none'; + fetchGroups(); + } catch (e) { console.error(e); } +} diff --git a/front-end/js/main.js b/front-end/js/main.js new file mode 100644 index 0000000..9c94f30 --- /dev/null +++ b/front-end/js/main.js @@ -0,0 +1,165 @@ +// ------------------------------------------------------------- +// Initialization and Global Event Listeners +// ------------------------------------------------------------- + +function switchTab(tabName) { + state.activeTab = tabName; + + // Update button active states + document.querySelectorAll('.tab-btn').forEach(btn => { + btn.classList.toggle('active', btn.dataset.tab === tabName); + }); + + // Show/hide views + Object.entries(views).forEach(([name, el]) => { + el.classList.toggle('active', name === tabName); + }); + + if (tabName === 'dashboard') { + fetchDocuments(); + } else if (tabName === 'groups' && state.user && state.user.role === 'admin') { + fetchGroups(); + } +} +window.switchTab = switchTab; + +function render() { + if (state.isAuthenticated) { + views.auth.classList.remove('active'); + tabNav.style.display = 'flex'; + + navActions.innerHTML = ` + + ${state.user.role.toUpperCase()} + + ${state.user.username} + + `; + + // Toggle admin tabs + document.querySelectorAll('.admin-only').forEach(el => { + el.style.display = state.user.role === 'admin' ? 'inline-block' : 'none'; + }); + + switchTab(state.activeTab); + } else { + Object.values(views).forEach(v => v.classList.remove('active')); + views.auth.classList.add('active'); + tabNav.style.display = 'none'; + navActions.innerHTML = ''; + } +} +window.render = render; + +// ------------------------------------------------------------- +// Event Listeners Registration +// ------------------------------------------------------------- +document.addEventListener('DOMContentLoaded', () => { + + // Chat form submit + document.getElementById('chat-form')?.addEventListener('submit', (e) => { + e.preventDefault(); + const query = document.getElementById('chat-input').value.trim(); + if (!query) return; + document.getElementById('chat-input').value = ''; + performChat(query); + }); + + // Auth form toggle + document.getElementById('auth-toggle')?.addEventListener('click', () => { + state.isLoginMode = !state.isLoginMode; + document.getElementById('auth-title').innerText = state.isLoginMode ? 'Вход' : 'Регистрация'; + document.getElementById('auth-submit').innerText = state.isLoginMode ? 'Войти' : 'Создать аккаунт'; + document.getElementById('auth-toggle').innerText = state.isLoginMode + ? "Нет аккаунта? Зарегистрируйтесь" + : 'Уже есть аккаунт? Войти'; + }); + + // Auth form submit + document.getElementById('auth-form')?.addEventListener('submit', async (e) => { + e.preventDefault(); + const btn = document.getElementById('auth-submit'); + btn.innerHTML = ''; + btn.disabled = true; + + const username = document.getElementById('username').value; + const password = document.getElementById('password').value; + + try { + if (state.isLoginMode) { + const fd = new FormData(); + fd.append('username', username); + fd.append('password', password); + const data = await fetch(`${API_URL}/auth/login`, { method: 'POST', body: fd }).then(async r => { + if (!r.ok) throw new Error((await r.json()).detail); + return r.json(); + }); + localStorage.setItem('access_token', data.access_token); + localStorage.setItem('refresh_token', data.refresh_token); + } else { + await fetch(`${API_URL}/auth/register`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username, password }), + }).then(async r => { + if (!r.ok) throw new Error((await r.json()).detail); + }); + + // Auto login after registration + const fd = new FormData(); + fd.append('username', username); + fd.append('password', password); + const data = await fetch(`${API_URL}/auth/login`, { method: 'POST', body: fd }).then(async r => { + if (!r.ok) throw new Error((await r.json()).detail); + return r.json(); + }); + localStorage.setItem('access_token', data.access_token); + localStorage.setItem('refresh_token', data.refresh_token); + showToast('Успешная регистрация!'); + } + + await checkAuth(); + showToast(`Добро пожаловать, ${state.user.username}!`); + } catch (e) { + showToast(e.message, 'error'); + } finally { + btn.innerHTML = state.isLoginMode ? 'Войти' : 'Создать аккаунт'; + btn.disabled = false; + } + }); + + // Tab navigation + document.querySelectorAll('.tab-btn').forEach(btn => { + btn.addEventListener('click', () => switchTab(btn.dataset.tab)); + }); + + // Search form submit + document.getElementById('search-form')?.addEventListener('submit', (e) => { + e.preventDefault(); + const query = document.getElementById('search-input').value.trim(); + if (!query) return; + performSearch(query); + }); + + // Create Group button + document.getElementById('create-group-btn')?.addEventListener('click', () => { + if (typeof openCreateGroupModal === 'function') { + openCreateGroupModal(); + } + }); + + // Esc to close modals + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { + if (typeof closeDocModal === 'function') closeDocModal(); + if (document.getElementById('upload-modal') && document.getElementById('upload-modal').style.display !== 'none' && typeof closeUploadModal === 'function') closeUploadModal(); + if (document.getElementById('group-modal') && document.getElementById('group-modal').style.display !== 'none') { + if (typeof closeDocAccessModal === 'function') closeDocAccessModal(); + else document.getElementById('group-modal').style.display = 'none'; + } + } + }); + + // Start App + checkAuth(); +}); diff --git a/front-end/js/search.js b/front-end/js/search.js new file mode 100644 index 0000000..b25d6bd --- /dev/null +++ b/front-end/js/search.js @@ -0,0 +1,87 @@ +// ------------------------------------------------------------- +// Search Functions +// ------------------------------------------------------------- + +/** + * Highlights query words in text by wrapping them in tags. + */ +function highlightText(text, query) { + if (!query) return escapeHtml(text); + const words = query.trim().split(/\s+/).filter(w => w.length >= 3); + if (!words.length) return escapeHtml(text); + + let escaped = escapeHtml(text); + words.forEach(word => { + const re = new RegExp(`(${word})`, 'gi'); + escaped = escaped.replace(re, '$1'); + }); + return escaped; +} + +function renderSearchResults(results, query) { + const container = document.getElementById('search-results'); + const stateEl = document.getElementById('search-state'); + + if (!results || results.length === 0) { + stateEl.style.display = 'block'; + stateEl.innerText = 'Ничего не найдено. Попробуйте другие ключевые слова.'; + container.innerHTML = ''; + return; + } + + stateEl.style.display = 'none'; + + container.innerHTML = results.map((r, i) => { + const extRaw = (r.extension || '').replace('.', '').toLowerCase(); + const extLabel = extRaw ? extRaw.toUpperCase() : '?'; + const scorePercent = Math.min(100, Math.round(r.score * 100)); + const keywords = (r.keywords || []).slice(0, 8); + const snippet = r.text.length > 400 ? r.text.slice(0, 400) + '…' : r.text; + const docIcon = EXT_ICONS[extRaw] || ''; + + return ` +
+
+
+ ${extLabel} + + ${docIcon} ${escapeHtml(r.document_title)} + +
+
+ Совпадение: ${scorePercent}% +
+
+
${highlightText(snippet, query)}
+ ${keywords.length ? ` +
+ ${keywords.map(kw => `${escapeHtml(kw)}`).join('')} +
` : ''} +
`; + }).join(''); +} + +async function performSearch(query) { + const stateEl = document.getElementById('search-state'); + const container = document.getElementById('search-results'); + + stateEl.style.display = 'block'; + stateEl.innerHTML = '  Поиск...'; + container.innerHTML = ''; + + try { + const results = await apiFetch('/documents/search', { + method: 'POST', + body: JSON.stringify({ query, top_k: 15 }), + }); + renderSearchResults(results, query); + } catch (e) { + stateEl.style.display = 'none'; + } +} +window.performSearch = performSearch; diff --git a/front-end/js/state.js b/front-end/js/state.js new file mode 100644 index 0000000..b5eb313 --- /dev/null +++ b/front-end/js/state.js @@ -0,0 +1,24 @@ +const API_URL = window.location.origin.startsWith('file') ? 'http://localhost:8000' : window.location.origin; + +// State +let state = { + isAuthenticated: false, + isLoginMode: true, + user: null, + documents: [], + groups: [], + users: [], + activeTab: 'dashboard', + docsPagination: { total: 0, page: 1, pages: 1, page_size: 12 }, +}; + +// DOM Elements +const views = { + auth: document.getElementById('auth-view'), + dashboard: document.getElementById('dashboard-view'), + search: document.getElementById('search-view'), + chat: document.getElementById('chat-view'), + groups: document.getElementById('groups-view'), +}; +const navActions = document.getElementById('nav-actions'); +const tabNav = document.getElementById('tab-nav'); diff --git a/front-end/js/theme.js b/front-end/js/theme.js new file mode 100644 index 0000000..d936a05 --- /dev/null +++ b/front-end/js/theme.js @@ -0,0 +1,35 @@ +// ------------------------------------------------------------- +// Theme Management +// ------------------------------------------------------------- + +function initTheme() { + const savedTheme = localStorage.getItem('app-theme'); + // Default to light theme if no saved theme + const themeToApply = savedTheme || 'light'; + document.documentElement.setAttribute('data-theme', themeToApply); + + window.addEventListener('DOMContentLoaded', () => { + const btn = document.getElementById('theme-toggle-btn'); + if (btn) { + btn.innerHTML = themeToApply === 'dark' ? 'Светлая' : 'Темная'; + } + }); +} + +function toggleTheme() { + const currentTheme = document.documentElement.getAttribute('data-theme'); + const newTheme = currentTheme === 'dark' ? 'light' : 'dark'; + document.documentElement.setAttribute('data-theme', newTheme); + localStorage.setItem('app-theme', newTheme); + + // Update button text if it exists + const btn = document.getElementById('theme-toggle-btn'); + if (btn) { + btn.innerHTML = newTheme === 'dark' ? 'Светлая' : 'Темная'; + } +} + +window.toggleTheme = toggleTheme; + +// Initialize theme immediately to prevent flash +initTheme(); diff --git a/front-end/js/utils.js b/front-end/js/utils.js new file mode 100644 index 0000000..11c4c8c --- /dev/null +++ b/front-end/js/utils.js @@ -0,0 +1,75 @@ +// ------------------------------------------------------------- +// Utility: Toasts +// ------------------------------------------------------------- +function showToast(message, type = 'success') { + const container = document.getElementById('toast-container'); + const toast = document.createElement('div'); + toast.className = `toast ${type}`; + toast.innerText = message; + container.appendChild(toast); + setTimeout(() => { + toast.style.opacity = '0'; + setTimeout(() => toast.remove(), 300); + }, 3000); +} + +// ------------------------------------------------------------- +// Utility: Formatting and HTML Escaping +// ------------------------------------------------------------- +function escapeHtml(str) { + if (!str) return ''; + return String(str) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function formatBytes(bytes, decimals = 2) { + if (!+bytes) return '0 Байт'; + const k = 1024, dm = decimals < 0 ? 0 : decimals; + const sizes = ['Байт', 'КБ', 'МБ', 'ГБ', 'ТБ']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`; +} + +// ------------------------------------------------------------- +// Utility: API Fetch with Token Interceptor +// ------------------------------------------------------------- +async function apiFetch(endpoint, options = {}) { + const token = localStorage.getItem('access_token'); + const headers = { ...options.headers }; + + if (token) { + headers['Authorization'] = `Bearer ${token}`; + } + + if (!(options.body instanceof FormData) && !headers['Content-Type']) { + headers['Content-Type'] = 'application/json'; + } + + try { + let response = await fetch(`${API_URL}${endpoint}`, { ...options, headers }); + + if (response.status === 401 && token) { + const refreshed = await refreshToken(); + if (refreshed) { + headers['Authorization'] = `Bearer ${localStorage.getItem('access_token')}`; + response = await fetch(`${API_URL}${endpoint}`, { ...options, headers }); + } else { + if (typeof logout === 'function') logout(); + throw new Error('Сессия истекла. Пожалуйста, войдите снова.'); + } + } + + if (!response.ok) { + const err = await response.json().catch(() => ({ detail: response.statusText })); + throw new Error(err.detail || 'Ошибка API'); + } + + return await response.json(); + } catch (error) { + showToast(error.message, 'error'); + throw error; + } +} diff --git a/keyword-extraction b/keyword-extraction new file mode 160000 index 0000000..48574fa --- /dev/null +++ b/keyword-extraction @@ -0,0 +1 @@ +Subproject commit 48574fa538e5cc308348d29efcb0e29e8d2f62d6 diff --git a/parser/assets/tests_results/extract_from_pdf_file.txt b/parser/assets/tests_results/extract_from_pdf_file.txt index e8d8167..d48176a 100644 --- a/parser/assets/tests_results/extract_from_pdf_file.txt +++ b/parser/assets/tests_results/extract_from_pdf_file.txt @@ -1,7 +1,7 @@ -КРИмеются следующие данные о величине товарооборота для 50 магазинов города ( Δ 𝑖 — -товарооборот, усл. руб.; 𝑛 𝑖 — число магазинов) +КРИмеются следующие данные о величине товарооборота для 50 магазинов города ( Δ 𝑖 - +товарооборот, усл. руб.; 𝑛 𝑖 - число магазинов) Δ 𝑖 [ 0 , 5 0 ) [ 5 0 , 1 0 0 ) [ 1 0 0 , 1 5 0 ) [ 1 5 0 , 2 0 0 ) [ 2 0 0 , 2 5 0 ) [ 2 5 0 , 3 0 0 ) diff --git a/parser/src/parsers/xml.rs b/parser/src/parsers/xml.rs index 809dd6d..587a238 100644 --- a/parser/src/parsers/xml.rs +++ b/parser/src/parsers/xml.rs @@ -40,10 +40,8 @@ pub(crate) fn get_info_from_xml_rels( match attr.key.as_ref() { b"Id" => id = Some(attr.unescape_value()?), b"Target" => target = Some(attr.unescape_value()?), - b"Type" => { - if attr.value.as_ref().ends_with(b"/image") { - is_image = true; - } + b"Type" if attr.value.as_ref().ends_with(b"/image") => { + is_image = true; } _ => {} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aa90c9e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +fastapi +uvicorn[standard] +sqlalchemy +asyncpg +alembic +python-dotenv +python-multipart +passlib[bcrypt] +python-jose[cryptography] +sentence-transformers +nltk +pymorphy3 +numpy +requests +qdrant-client \ No newline at end of file diff --git a/scripts/init_db.py b/scripts/init_db.py new file mode 100644 index 0000000..4a15128 --- /dev/null +++ b/scripts/init_db.py @@ -0,0 +1,21 @@ +import asyncio +import sys +import os + +# Добавляем текущую директорию в путь +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Импортируем БД модули +from app.database import engine, Base +from app.models import * + +async def init_db(): + async with engine.begin() as conn: + # Удаляем все таблицы (для чистой инициализации) + await conn.run_sync(Base.metadata.drop_all) + # Создаём заново + await conn.run_sync(Base.metadata.create_all) + print("Tables created successfully") + +if __name__ == "__main__": + asyncio.run(init_db()) \ No newline at end of file diff --git a/scripts/wait_for_db.py b/scripts/wait_for_db.py new file mode 100644 index 0000000..05ba7c5 --- /dev/null +++ b/scripts/wait_for_db.py @@ -0,0 +1,24 @@ +import asyncio +import asyncpg +import os +import sys + +async def wait_for_db(): + db_url = os.getenv("DATABASE_URL") + if not db_url: + print("DATABASE_URL not set") + sys.exit(1) + # преобразуем postgresql+asyncpg:// -> postgresql:// + dsn = db_url.replace("postgresql+asyncpg://", "postgresql://") + while True: + try: + conn = await asyncpg.connect(dsn) + await conn.close() + print("Database is ready") + break + except Exception as e: + print(f"Waiting for database... {e}") + await asyncio.sleep(1) + +if __name__ == "__main__": + asyncio.run(wait_for_db()) \ No newline at end of file diff --git a/test_qdrant.py b/test_qdrant.py new file mode 100644 index 0000000..25b248e --- /dev/null +++ b/test_qdrant.py @@ -0,0 +1,43 @@ +import asyncio +import uuid +from app.embeddings import get_embedding +from app.qdrant_client import index_chunk, semantic_search, get_chunk_count, delete_all_chunks + +async def test(): + print("=== Testing Qdrant ===\n") + + # Очистка + print("1. Cleaning up...") + delete_all_chunks() + print(f" Chunks after cleanup: {get_chunk_count()}\n") + + # Индексация тестового чанка + print("2. Indexing test chunk...") + chunk_id = str(uuid.uuid4()) + text = "Нейронные сети и машинное обучение" + emb = get_embedding(text) + success = index_chunk( + chunk_id=chunk_id, + document_id="test-doc-123", + chunk_index=0, + text=text, + keywords=["нейронные сети", "машинное обучение"], + language="ru", + embedding=emb + ) + print(f" Indexing success: {success}") + print(f" Chunks after index: {get_chunk_count()}\n") + + # Семантический поиск + print("3. Semantic search...") + query = "глубокое обучение" + query_emb = get_embedding(query) + results = semantic_search(query_emb, top_k=3) + print(f" Found {len(results)} results") + for r in results: + print(f" - Score: {r.get('score', 0):.4f} | Text: {r.get('text', '')[:50]}...") + + print("\n✓ Test completed") + +if __name__ == "__main__": + asyncio.run(test()) \ No newline at end of file