diff --git a/.env b/.env index 789860c064f..d015689eabe 100644 --- a/.env +++ b/.env @@ -48,6 +48,8 @@ _APP_CONSOLE_GOOGLE_APP_ID= _APP_CONSOLE_GOOGLE_SECRET= # MariaDB and MongoDB only start when their profile is listed here; PostgreSQL always # runs because VectorsDB requires it. MongoDB is on by default for DocumentsDB. +# The embedding container is resource-heavy and off by default; add 'embedding' here to +# start it. The embeddings API is on by default and is switched off below until then. COMPOSE_PROFILES=mongodb _APP_DB_ADAPTER=postgresql _APP_DB_HOST=postgresql @@ -68,6 +70,7 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_USER= _APP_REDIS_PASS= +_APP_EMBEDDING=disabled _APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:3000/embed' _APP_EMBEDDING_MODELS=nomic-embed-text _APP_EMBEDDING_TIMEOUT=30000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d550cbde3fd..5ba277f7d8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,7 +196,7 @@ jobs: { name: 'Account', runner: runner4, functional: true, processes: null }, { name: 'Avatars', runner: runner4, functional: true, processes: null }, { name: 'Console', runner: runner4, functional: true, processes: null }, - { name: 'Databases', runner: runner8, functional: false, processes: 3, documentsdb: true }, + { name: 'Databases', runner: runner8, functional: false, processes: 3, documentsdb: true, embedding: true }, { name: 'TablesDB', runner: runner8, functional: false, processes: 3 }, { name: 'Functions', runner: runner4, functional: false, processes: null }, { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null }, @@ -469,6 +469,13 @@ jobs: COMPOSE_PROFILES="${COMPOSE_PROFILES},gitea" fi + # The embedding container is resource-heavy and off by default, so start it and + # enable the API only for the suites that exercise the embeddings routes. + if [ "${{ matrix.service.embedding }}" = "true" ]; then + COMPOSE_PROFILES="${COMPOSE_PROFILES},embedding" + echo "_APP_EMBEDDING=enabled" >> $GITHUB_ENV + fi + echo "COMPOSE_PROFILES=${COMPOSE_PROFILES}" >> $GITHUB_ENV - name: Login to Docker Hub diff --git a/app/config/variables.php b/app/config/variables.php index 96f21380f22..9b758e39f48 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -715,6 +715,15 @@ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_EMBEDDING', + 'description' => 'Enables the embeddings API, backed by the resource-heavy appwrite-embedding container. That container sits behind the "embedding" Compose profile, so add "embedding" to COMPOSE_PROFILES to start it. Set this to "disabled" to have the /v1/embeddings routes return a service disabled error instead of reaching for the container. Default value is: enabled.', + 'introduction' => '2.0.0', + 'default' => 'enabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 53bb02dd497..96d1ccd7d5f 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -465,14 +465,18 @@ // installation deploys just the engine backing the platform, so neither is on // until an operator provisions that engine and says so. Closed to everyone -- // keys and privileged roles included -- rather than answering and then failing - // on the first write with the reason only in the logs. + // against an absent service with the reason only in the logs. Embeddings ran + // on every installation before it had a switch, so it stays on unless an + // operator turns it off; the resource-heavy container is what sits behind a + // Compose profile. $products = [ - 'documentsdb' => '_APP_DOCUMENTSDB', - 'vectorsdb' => '_APP_VECTORSDB', + 'documentsdb' => ['_APP_DOCUMENTSDB', 'disabled'], + 'vectorsdb' => ['_APP_VECTORSDB', 'disabled'], + 'embeddings' => ['_APP_EMBEDDING', 'enabled'], ]; if ( isset($products[$namespace]) - && System::getEnv($products[$namespace], 'disabled') !== 'enabled' + && System::getEnv(...$products[$namespace]) !== 'enabled' ) { throw new Exception(Exception::GENERAL_SERVICE_DISABLED); } diff --git a/docker-compose.yml b/docker-compose.yml index a047ceb583f..b7dc3e4fb98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,7 +75,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding - appwrite-geo environment: - _APP_ENV @@ -260,6 +259,7 @@ services: - _APP_LIMIT_DATABASE_BATCH - _APP_DOCUMENTSDB - _APP_VECTORSDB + - _APP_EMBEDDING appwrite-console: logging: driver: json-file @@ -314,7 +314,6 @@ services: depends_on: - ${_APP_DB_HOST:-postgresql} - redis - - appwrite-embedding environment: - _APP_ENV - _APP_USAGE_STATS @@ -356,6 +355,7 @@ services: - _APP_LIMIT_DATABASE_BATCH - _APP_DOCUMENTSDB - _APP_VECTORSDB + - _APP_EMBEDDING - _APP_POOL_ADAPTER=swoole appwrite-worker: @@ -373,7 +373,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -525,6 +524,7 @@ services: - _APP_LIMIT_DATABASE_BATCH - _APP_DOCUMENTSDB - _APP_VECTORSDB + - _APP_EMBEDDING appwrite-task-scheduler: entrypoint: schedule @@ -541,7 +541,6 @@ services: depends_on: - ${_APP_DB_HOST:-postgresql} - redis - - appwrite-embedding environment: - _APP_ENV - _APP_USAGE_STATS @@ -622,7 +621,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding environment: - _APP_ENV - _APP_WORKERS_NUM=1 @@ -662,7 +660,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -753,7 +750,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding environment: - _APP_ENV - _APP_WORKERS_NUM=1 @@ -792,6 +788,7 @@ services: - _APP_LIMIT_DATABASE_BATCH - _APP_DOCUMENTSDB - _APP_VECTORSDB + - _APP_EMBEDDING appwrite-worker-builds: profiles: - separate @@ -812,7 +809,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding environment: - _APP_ENV - _APP_WORKERS_NUM=1 @@ -1043,7 +1039,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -1093,7 +1088,6 @@ services: depends_on: - redis - ${_APP_DB_HOST:-postgresql} - - appwrite-embedding environment: - _APP_ENV - _APP_USAGE_PASS @@ -1498,7 +1492,6 @@ services: depends_on: - ${_APP_DB_HOST:-postgresql} - redis - - appwrite-embedding environment: - _APP_ENV - _APP_LOGGING_FORMAT @@ -1541,7 +1534,6 @@ services: depends_on: - ${_APP_DB_HOST:-postgresql} - redis - - appwrite-embedding environment: - _APP_ENV - _APP_LOGGING_FORMAT @@ -1583,7 +1575,6 @@ services: depends_on: - ${_APP_DB_HOST:-postgresql} - redis - - appwrite-embedding environment: - _APP_ENV - _APP_LOGGING_FORMAT @@ -1838,6 +1829,8 @@ services: - postgres appwrite-embedding: image: appwrite/embedding:0.1.0 + profiles: + - embedding environment: EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic-embed-text} container_name: appwrite-embedding