diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index fe050447..45c950ba 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -73,7 +73,10 @@ jobs: run: make dev.provision.${{matrix.services}} - name: "Bring up services" - run: make dev.up.${{matrix.services}} + # Use the without-wait variant: this job only provisions a subset of + # services, so their un-provisioned transitive dependencies can never + # become healthy. + run: make dev.up.without-wait.${{matrix.services}} - name: "Wait for services to become ready" run: | diff --git a/Makefile b/Makefile index c06e37b9..b0ff7eaa 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,8 @@ dev.shell.cms_watcher dev.shell.xqueue dev.shell.xqueue_consumer \ dev.static dev.static.lms dev.static.cms dev.stats dev.status \ dev.stop dev.up dev.up.attach dev.up.shell \ - dev.up.without-deps dev.up.without-deps.shell dev.up.with-programs \ + dev.up.without-deps dev.up.without-deps.shell dev.up.without-wait \ + dev.up.with-programs \ dev.up.with-watchers dev.validate docs \ help requirements impl-dev.clone.https impl-dev.clone.ssh impl-dev.provision \ impl-dev.pull impl-dev.pull.without-deps impl-dev.up impl-dev.up.attach \ @@ -254,7 +255,7 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc dev.up.without-deps: _expects-service-list.dev.up.without-deps dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves. - docker compose up -d --no-deps $$(echo $* | tr + " ") + docker compose up -d --wait --no-deps $$(echo $* | tr + " ") dev.up.without-deps.shell: _expects-service.dev.up.without-deps.shell @@ -262,6 +263,11 @@ dev.up.without-deps.shell.%: ## Bring up a service by itself + shell into it. make dev.up.without-deps.$* make dev.shell.$* +dev.up.without-wait: _expects-service-list.dev.up.without-wait + +dev.up.without-wait.%: dev.check-memory ## Bring up services and their dependencies without waiting for them to become healthy. + docker compose up -d $$(echo $* | tr + " ") + dev.up: @scripts/make_warn_default_large.sh "$@" @@ -269,7 +275,7 @@ dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services. @echo # at least one statement so that dev.up.% doesn't run too dev.up.%: dev.check-memory ## Bring up services and their dependencies. - docker compose up -d $$(echo $* | tr + " ") + docker compose up -d --wait $$(echo $* | tr + " ") ifeq ($(ALWAYS_CACHE_PROGRAMS),true) make dev.cache-programs endif @@ -552,9 +558,6 @@ validate-lms-volume: ## Validate that changes to the local workspace are reflect docker compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile -hadoop-application-logs-%: ## View hadoop logs by application Id. - docker compose exec nodemanager yarn logs -applicationId $* - create-test-course: ## Provisions cms, and ecommerce with course(s) in test-course.json. bash ./course-generator/create-courses.sh --cms --ecommerce course-generator/test-course.json diff --git a/check.sh b/check.sh index 7e95a858..03aa59d0 100755 --- a/check.sh +++ b/check.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Run checks for the provided service(s). +# Run health checks for the provided service(s). # To specify multiple services, separate them with spaces or plus signs (+). -# To specify all services, just pass in "all". +# To specify all running services, just pass in "all". # # Examples: # ./check.sh lms @@ -9,7 +9,7 @@ # ./check.sh lms+forum discovery # ./check.sh all # -# Exists 0 if successful; non-zero otherwise. +# Exits 0 if successful; non-zero otherwise. # # Fails if no services specified. # @@ -18,25 +18,11 @@ set -eu -o pipefail -# Grab all arguments into one string, replacing plus signs with spaces. -# Pad on either side with spaces so that the regex in `should_check` works correctly. -services=" ${*//+/ } " - # Which checks succeeded and failed. succeeded="" failed="" -# Returns whether service in first arg should be checked. -should_check() { - local service="$1" - if [[ "$services" == *" all "* ]] || [[ "$services" == *" $service "* ]]; then - return 0 # Note that '0' means 'success' (i.e., true) in bash. - else - return 1 - fi -} - -# Runs a check named $1 on service $2 using the command $3. +# Runs a check named $1 on service $2 using the host-side command $3. run_check() { local check_name="$1" local service="$2" @@ -53,133 +39,75 @@ run_check() { echo # Newline } -mysql_run_check() { - container_name="$1" - mysql_probe="SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" - # The use of `--protocol tcp` forces MySQL to connect over TCP rather than - # via a UNIX socket. This is needed because when MySQL starts for the first - # time in a new container, it starts a "temporary server" that runs for a - # few seconds and then shuts down before the "real" server starts up. The - # temporary server does not listen on the TCP port, but if the mysql - # command is not told which server to use, it will first try the UNIX - # socket and only after that will it try the default TCP port. - # - # By specifying that mysql should use TCP, we won't get an early false - # positive "ready" response while the temporary server is running. - run_check "${container_name}_query" "$container_name" \ - "docker compose exec -T $(printf %q "$container_name") mysql --protocol tcp -uroot -se $(printf %q "$mysql_probe")" +# Print a service container's Docker healthcheck status, one of: +# healthy | unhealthy | starting | none | missing +# "none" means the container exists but declares no healthcheck; "missing" +# means no container is running for the service. +container_health() { + local service="$1" cid + cid="$(docker compose ps -q "$service" 2>/dev/null || true)" + if [[ -z "$cid" ]]; then + echo "missing" + return + fi + docker inspect \ + --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + "$cid" 2>/dev/null || echo "missing" } -if should_check mysql80; then - echo "Checking MySQL 8.0 query endpoint:" - mysql_run_check mysql80 -fi - -if should_check mongo; then - echo "Checking MongoDB status:" - run_check mongo_status mongo \ - "docker compose exec -T mongo mongo --eval \"db.serverStatus()\"" -fi - -if should_check registrar; then - echo "Checking Registrar heartbeat:" - run_check registrar_heartbeat registrar \ - "curl --fail -L http://localhost:18734/health" -fi - -if should_check lms; then - echo "Checking LMS heartbeat:" - run_check lms_heartbeat lms \ - "curl --fail -L http://localhost:18000/heartbeat" - - echo "Validating LMS volume:" - run_check lms_volume lms \ - "make validate-lms-volume" -fi - -if should_check cms; then - echo "Checking CMS heartbeat:" - run_check cms_heartbeat cms \ - "curl --fail -L http://localhost:18010/heartbeat" -fi - -if should_check ecommerce; then - echo "Checking ecommerce health:" - run_check ecommerce_heartbeat ecommerce \ - "curl --fail -L http://localhost:18130/health/" -fi - -if should_check enterprise_access; then - echo "Checking enterprise-access health:" - run_check enterprise_access_heartbeat enterprise-access \ - "curl --fail -L http://localhost:18130/health/" -fi - -if should_check enterprise-subsidy; then - echo "Checking enterprise_subsidy health:" - run_check enterprise-subsidy_heartbeat enterprise-subsidy \ - "curl --fail -L http://localhost:18280/health/" -fi - -if should_check discovery; then - echo "Checking discovery health:" - run_check discovery_heartbeat discovery \ - "curl --fail -L http://localhost:18381/health/" -fi - -if should_check forum; then - echo "Checking forum heartbeat:" - run_check forum_heartbeat forum \ - "curl --fail -L http://localhost:44567/heartbeat" -fi - -if should_check edx_notes_api; then - echo "Checking edx_notes_api heartbeat:" - run_check edx_notes_api_heartbeat edx_notes_api \ - "curl --fail -L http://localhost:18120/heartbeat" -fi - -if should_check designer; then - echo "Checking designer health:" - run_check designer_heartbeat designer \ - "curl --fail -L http://localhost:18808/health/" -fi - -if should_check credentials; then - echo "Checking credentials heartbeat:" - run_check credentials_heartbeat credentials \ - "curl --fail -L http://localhost:18150/health" -fi - -if should_check xqueue; then - echo "Checking xqueue status:" - run_check xqueue_heartbeat xqueue \ - "curl --fail -L http://localhost:18040/xqueue/status" -fi - -if should_check insights; then - echo "Running Analytics Dashboard Devstack tests: " - run_check insights_heartbeat insights \ - "curl --fail -L http://localhost:18110/health/" -fi +# Default check: pass iff the service's container reports itself healthy. +# Services with no healthcheck (and no extra check) simply don't contribute a +# check, matching the old behavior for unrecognized services. +run_health_check() { + local service="$1" + local status + status="$(container_health "$service")" + case "$status" in + healthy) + echo "Checking $service: healthy" + succeeded="$succeeded ${service}_health" + echo + ;; + starting|unhealthy) + echo "Checking $service: $status" + docker compose logs --tail 500 "$service" + failed="$failed ${service}_health" + echo + ;; + none|missing) + # No container healthcheck to consult; rely on any extra checks. + : + ;; + esac +} -if should_check analyticsapi; then - echo "Running Analytics Data API Devstack tests: " - run_check analyticsapi_heartbeat analyticsapi \ - "curl --fail -L http://localhost:19001/health/" -fi +# Extra/override checks for services that need more than their container +# healthcheck can express. Keep this SMALL -- it is the only place service +# names should be enumerated. +run_extra_checks() { + local service="$1" + case "$service" in + lms) + echo "Validating LMS volume:" + run_check lms_volume lms "make validate-lms-volume" + ;; + esac +} -if should_check license-manager; then - echo "Running License Manager Devstack tests: " - run_check license_manager_heartbeat license-manager \ - "curl --fail -L http://localhost:18170/health/" +# Expand the requested services into a plain, space-separated list. "all" +# means every service with a running container (word-splitting is safe here +# because compose service names contain no whitespace). +requested=" ${*//+/ } " +if [[ "$requested" == *" all "* ]]; then + service_list="$(docker compose ps --services)" +else + service_list="${*//+/ }" fi -if should_check edx-exams; then - echo "Running edX Exam Devstack tests: " - run_check edx-exams_heartbeat edx-exams \ - "curl --fail -L http://localhost:18740/health/" -fi +for service in $service_list; do + run_health_check "$service" + run_extra_checks "$service" +done echo "Successful checks:${succeeded:- NONE}" echo "Failed checks:${failed:- NONE}" diff --git a/common.yml b/common.yml new file mode 100644 index 00000000..a38ca489 --- /dev/null +++ b/common.yml @@ -0,0 +1,87 @@ +# Abstract base services, inherited by concrete services via `extends`. +services: + + # Traffic-serving backend app (Django/IDA) services. + backend-app: + stdin_open: true # Allows `make dev.attach.` to work correctly. + tty: true + # Concrete services may set these extra environment variables: + # + # - DEVSERVER_SETUP allows some services to run extra commands at the very beginning. + # This is rarely needed. + # - DEVSERVER_CMD runs inside an infinite loop to allow `make dev.restart-devserver.` + # to work correctly. + # + # Examples: + # + # environment: + # DEVSERVER_SETUP: "source /edx/app/foo/foo_env" # optional + # DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18000" + command: + - bash + - -c + - | + if [ -n "$$DEVSERVER_SETUP" ]; then eval "$$DEVSERVER_SETUP"; fi + while true; do + eval "$$DEVSERVER_CMD" + sleep 2 + done + # Configure a healthcheck to allow `make dev.up.` to give accurate feedback + # about whether the service actually started successfully and is serving traffic. + # + # Set HEALTHCHECK_TARGET to the full URL curl should hit, e.g.: + # + # environment: + # HEALTHCHECK_TARGET: "http://localhost:18150/health/" + # + # Timings are sized so the worst case from container start to "give up" + # (marked unhealthy) is ~5 minutes: start_period + retries*interval = 300s = 5min. + healthcheck: + test: ["CMD-SHELL", "curl -fsS $$HEALTHCHECK_TARGET || exit 1"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 240s + start_interval: 5s # poll fast during grace period so a healthy boot flips status quickly + + # All microfrontends. + microfrontend: + # Use `npm ci` rather than `npm install` for a few reasons: + # + # - Repeatability: Respect the currently checked out package + # versions rather than upgrading when package.json and + # package-lock.json don't match. (Two people using this at + # different times on the same commit should get the same + # results.) + # - Immutability: Don't change the repo's working directory + # unexpectedly when there's a lock mismatch. + # + # Fail fast if package install fails to avoid mysterious + # errors later. + command: + - bash + - -c + - | + npm ci || exit 1 + if [ -n "$${PARAGON_BRAND_PACKAGE}" ]; then + npx paragon install-theme "$${PARAGON_BRAND_PACKAGE}" || exit 1 + fi + while true; do + npm start + sleep 2 + done + stdin_open: true + tty: true + image: node:18 + environment: + - NODE_ENV=development + # MFEs take longer to startup: a cold `npm ci` + `npm start` is empirically + # observed to reach healthy in about 5 minutes, so we shouldn't give up + # until at least ~10 minutes. start_period + retries*interval = 600s = 10min. + healthcheck: + test: ["CMD-SHELL", "curl -fsS $$HEALTHCHECK_TARGET || exit 1"] + interval: 10s + timeout: 10s + retries: 6 + start_period: 540s + start_interval: 5s # poll fast during grace period so a healthy boot flips status quickly diff --git a/docker-compose.yml b/docker-compose.yml index 43c60187..e8d11cf0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,7 +78,8 @@ services: hostname: kafka.devstack.edx container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.kafka" depends_on: - - zookeeper + zookeeper: + condition: service_started ports: - "9092:9092" - "9101:9101" @@ -112,8 +113,10 @@ services: hostname: kafka-control-center.devstack.edx container_name: edx.${COMPOSE_PROJECT_NAME:-devstack}.kafka-control-center depends_on: - - kafka - - schema-registry + kafka: + condition: service_started + schema-registry: + condition: service_started ports: - "9021:9021" environment: @@ -146,6 +149,14 @@ services: # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --nojournal --storageEngine wiredTiger container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" + healthcheck: + # Exit 0 only if the server responds to a ping. `mongo` is the legacy + # shell shipped in the 5.0 image (mongosh arrived in 6.0). + test: ["CMD-SHELL", "mongo --quiet --eval 'quit(db.adminCommand({ ping: 1 }).ok ? 0 : 1)'"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 60s hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-5.0.24} networks: @@ -161,10 +172,10 @@ services: mysql80: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql80" - # Use --protocol=tcp so the check waits for the real server on port 3306, not the - # brief temporary server MySQL runs on the UNIX socket during first-time container - # initialization. See mysql_run_check() in check.sh for context. healthcheck: + # Use --protocol=tcp so the check waits for the real server on port 3306, not the + # brief temporary server MySQL runs on the UNIX socket during first-time container + # initialization. test: ["CMD", "mysqladmin", "ping", "--protocol=tcp", "-uroot"] interval: 5s timeout: 3s @@ -204,7 +215,8 @@ services: hostname: schema-registry.devstack.edx container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.schema-registry" depends_on: - - kafka + kafka: + condition: service_started ports: - "8081:8081" environment: @@ -237,7 +249,9 @@ services: # ================================================ credentials: - command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" hostname: credentials.devstack.edx depends_on: @@ -247,10 +261,9 @@ services: condition: service_started mysql80: condition: service_healthy - # Allows attachment to the credentials service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18150" + HEALTHCHECK_TARGET: "http://localhost:18150/health/" CACHE_LOCATION: edx.devstack.memcached:11211 DB_HOST: edx.devstack.mysql80 SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2 @@ -265,7 +278,9 @@ services: - "18150:18150" discovery: - command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" hostname: discovery.devstack.edx depends_on: @@ -279,17 +294,16 @@ services: condition: service_started redis: condition: service_started - # Allows attachment to the discovery service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18381" + HEALTHCHECK_TARGET: "http://localhost:18381/health/" # This next DB_MIGRATION_HOST line can be removed once edx/configuration has been updated with this value for # a while and most people have had a chance to do a "make pull" to get the latest images. DB_MIGRATION_HOST: edx.devstack.mysql80 TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710" ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 - image: edxops/discovery-dev:latest + image: edxops/course-discovery-dev:latest networks: default: aliases: @@ -301,7 +315,9 @@ services: - ${PWD}/configuration_files/discovery.yml:/edx/etc/discovery.yml ecommerce: - command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" hostname: ecommerce.devstack.edx depends_on: @@ -313,10 +329,9 @@ services: condition: service_started mysql80: condition: service_healthy - # Allows attachment to the ecommerce service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18130" + HEALTHCHECK_TARGET: "http://localhost:18130/health/" DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce-dev:latest @@ -331,8 +346,9 @@ services: edx_notes_api: - # Sleep as a part of start up to give elasticsearch enough time to start up. - command: bash -c 'while true; do python /edx/app/notes/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 4; done' + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edxnotesapi" hostname: edx_notes_api.devstack.edx depends_on: @@ -350,6 +366,8 @@ services: ports: - "18120:18120" environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18120" + HEALTHCHECK_TARGET: "http://localhost:18120/heartbeat/" DB_ENGINE: "django.db.backends.mysql" DB_HOST: "edx.devstack.mysql80" DB_NAME: "notes" @@ -362,10 +380,12 @@ services: ELASTICSEARCH_DSL: "http://edx.devstack.elasticsearch710:9200" enterprise-access: + extends: + file: common.yml + service: backend-app image: edxops/enterprise-access-dev container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.enterprise-access" hostname: enterprise-access.devstack.edx - command: bash -c 'while true; do python /edx/app/enterprise-access/manage.py runserver 0.0.0.0:18270; sleep 2; done' ports: - "18270:18270" depends_on: @@ -379,9 +399,9 @@ services: default: aliases: - edx.devstack.enterprise-access - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18270" + HEALTHCHECK_TARGET: "http://localhost:18270/health/" CELERY_ALWAYS_EAGER: 'false' CELERY_BROKER_TRANSPORT: redis CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 @@ -430,17 +450,25 @@ services: tty: true forum: - command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ./bin/unicorn -c config/unicorn_tcp.rb -I .; sleep 2; done' + extends: + file: common.yml + service: backend-app + environment: + DEVSERVER_CMD: "./bin/unicorn -c config/unicorn_tcp.rb -I ." + DEVSERVER_SETUP: "source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install" + HEALTHCHECK_TARGET: "http://localhost:4567/heartbeat" container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" hostname: forum.devstack.edx depends_on: - - memcached - - mongo - - elasticsearch710 - - opensearch12 + memcached: + condition: service_started + mongo: + condition: service_started + elasticsearch710: + condition: service_started + opensearch12: + condition: service_started image: edxops/forum:latest - stdin_open: true - tty: true networks: default: aliases: @@ -449,10 +477,16 @@ services: - "44567:4567" lms: + extends: + file: common.yml + service: backend-app # Switch to `--settings devstack_with_worker` if you want to use lms-worker - command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx + # "healthy" == the runserver process is actually accepting HTTP requests on + # :18000 and its DB/cache/modulestore checks pass. /heartbeat does not depend + # on migrations having run, so this won't block a freshly-created devstack. + # start_period is generous: first boot runs `pip install` before runserver. depends_on: discovery: condition: service_started @@ -466,10 +500,10 @@ services: condition: service_started mysql80: condition: service_healthy - # Allows attachment to the LMS service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py lms runserver 0.0.0.0:18000 --settings devstack" + DEVSERVER_SETUP: "pip install -e . -r /edx/private_requirements.txt" + HEALTHCHECK_TARGET: "http://localhost:18000/heartbeat" FRONTEND_TEST_SERVER_HOSTNAME: edx.devstack.lms FRONTEND_TEST_SERVER_LMS_PORT: 18003 FRONTEND_TEST_SERVER_CMS_PORT: 18031 @@ -513,7 +547,9 @@ services: C_FORCE_ROOT: "true" insights: - command: bash -c 'source /edx/app/insights/insights_env && while true; do python /edx/app/insights/insights/manage.py runserver 0.0.0.0:18110 --settings analytics_dashboard.settings.devstack; sleep 2; done' + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights" hostname: insights.devstack.edx depends_on: @@ -525,10 +561,9 @@ services: condition: service_started memcached: condition: service_started - # Allows attachment to the insights service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18110" + HEALTHCHECK_TARGET: "http://localhost:18110/health/" DB_HOST: edx.devstack.mysql80 DB_NAME: dashboard DB_PORT: 3306 @@ -550,6 +585,9 @@ services: - ${PWD}/configuration_files/insights.yml:/edx/etc/insights.yml analyticsapi: + extends: + file: common.yml + service: backend-app image: edxops/edx-analytics-data-api-dev:latest container_name: edx.devstack.analyticsapi hostname: analyticsapi @@ -558,10 +596,9 @@ services: condition: service_healthy elasticsearch710: condition: service_started - command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true; do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done' - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:19001" + HEALTHCHECK_TARGET: "http://localhost:19001/health/" DB_HOST: edx.devstack.mysql80 DB_PORT: 3306 DB_USER: analytics001 @@ -575,7 +612,9 @@ services: - ${PWD}/configuration_files/analytics_api.yml:/edx/etc/analytics_api.yml registrar: - command: bash -c 'while true; do python /edx/app/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" hostname: registrar.devstack.edx depends_on: @@ -591,10 +630,9 @@ services: condition: service_started registrar-worker: condition: service_started - # Allows attachment to the registrar service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18734" + HEALTHCHECK_TARGET: "http://localhost:18734/health/" DB_HOST: edx.devstack.mysql80 DB_NAME: registrar DB_PORT: 3306 @@ -661,10 +699,13 @@ services: - /edx/var/registrar/ cms: + extends: + file: common.yml + service: backend-app # Switch to `--settings devstack_with_worker` if you want to use cms-worker - command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack; sleep 2; done)' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms" hostname: cms.devstack.edx + # See the lms healthcheck for rationale; cms serves on :18010. depends_on: elasticsearch710: condition: service_started @@ -676,10 +717,10 @@ services: condition: service_started mysql80: condition: service_healthy - # Allows attachment to the CMS service using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py cms runserver 0.0.0.0:18010 --settings devstack" + DEVSERVER_SETUP: "pip install -e . -r /edx/private_requirements.txt" + HEALTHCHECK_TARGET: "http://localhost:18010/heartbeat" FRONTEND_TEST_SERVER_HOSTNAME: edx.devstack.cms FRONTEND_TEST_SERVER_LMS_PORT: 18103 FRONTEND_TEST_SERVER_CMS_PORT: 18131 @@ -723,13 +764,17 @@ services: C_FORCE_ROOT: "true" codejail: - command: bash -c 'source /venv/bin/activate; while true; do python ./manage.py runserver 0.0.0.0:8080; sleep 2; done' + extends: + file: common.yml + service: backend-app + healthcheck: + test: ["CMD-SHELL", "python3 -c \"import urllib.request; urllib.request.urlopen('http://localhost:8080/health/')\" || exit 1"] container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.codejail" hostname: codejail.devstack.edx - stdin_open: true - tty: true image: edxops/codejail-service-dev:latest environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:8080" + DEVSERVER_SETUP: "source /venv/bin/activate" DJANGO_SETTINGS_MODULE: codejail_service.settings.devstack ports: - "18030:8080" @@ -738,10 +783,12 @@ services: - apparmor=openedx_codejail_service xqueue: + extends: + file: common.yml + service: backend-app container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue-dev:latest working_dir: /edx/app/xqueue/xqueue - command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml @@ -749,6 +796,8 @@ services: mysql80: condition: service_healthy environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18040" + HEALTHCHECK_TARGET: "http://localhost:18040/xqueue/status/" XQUEUE_CFG: "/edx/etc/xqueue.yml" networks: default: @@ -759,6 +808,9 @@ services: xqueue_consumer: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" + # Allows attachment to the xqueue_consumer service using 'docker attach '. + stdin_open: true + tty: true image: edxops/xqueue-dev:latest working_dir: /edx/app/xqueue/xqueue command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' @@ -774,10 +826,12 @@ services: - edx.devstack.xqueue_consumer enterprise-catalog: + extends: + file: common.yml + service: backend-app image: edxops/enterprise-catalog-dev container_name: enterprise.catalog.app hostname: enterprise.catalog.app - command: bash -c 'while true; do python /edx/app/enterprise-catalog/manage.py runserver 0.0.0.0:18160; sleep 2; done' ports: - "18160:18160" depends_on: @@ -789,12 +843,12 @@ services: condition: service_started networks: default: - aliases: - - edx.devstack.enterprise-catalog + aliases: + - edx.devstack.enterprise-catalog # Allows attachment to this container using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18160" + HEALTHCHECK_TARGET: "http://localhost:18160/health/" CELERY_ALWAYS_EAGER: 'false' CELERY_BROKER_TRANSPORT: redis CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 @@ -857,11 +911,13 @@ services: tty: true license-manager: + extends: + file: common.yml + service: backend-app image: edxops/license-manager-dev:latest container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.license-manager" hostname: license-manager.devstack.edx # Use the Django devserver, so that we can hot-reload code changes - command: bash -c 'while true; do python /edx/app/license_manager/manage.py runserver 0.0.0.0:18170; sleep 2; done' ports: - "18170:18170" depends_on: @@ -870,9 +926,9 @@ services: license-manager-worker: condition: service_started # Allows attachment to this container using 'docker attach '. - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18170" + HEALTHCHECK_TARGET: "http://localhost:18170/health/" CELERY_ALWAYS_EAGER: 'false' CELERY_BROKER_TRANSPORT: redis CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 @@ -927,6 +983,9 @@ services: tty: true enterprise-subsidy: + extends: + file: common.yml + service: backend-app image: edxops/enterprise-subsidy-dev container_name: edx.devstack.enterprise-subsidy hostname: enterprise-subsidy.devstack.edx @@ -935,10 +994,9 @@ services: condition: service_healthy memcached: condition: service_started - command: bash -c 'while true; do python /edx/app/enterprise-subsidy/manage.py runserver 0.0.0.0:18280; sleep 2; done' - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18280" + HEALTHCHECK_TARGET: "http://localhost:18280/health/" DB_HOST: edx.devstack.mysql80 DB_PORT: 3306 DB_USER: subsidy001 @@ -970,6 +1028,9 @@ services: stdin_open: true edx-exams: + extends: + file: common.yml + service: backend-app image: edxops/edx-exams-dev container_name: edx.devstack.edx_exams hostname: edx_exams.devstack.edx @@ -978,10 +1039,9 @@ services: condition: service_started mysql80: condition: service_healthy - command: bash -c 'while true; do python /edx/app/edx-exams/manage.py runserver 0.0.0.0:18740; sleep 2; done' - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18740" + HEALTHCHECK_TARGET: "http://localhost:18740/health/" DB_HOST: edx.devstack.mysql80 DB_PORT: 3306 DB_NAME: edx_exams @@ -997,18 +1057,20 @@ services: # ========================================================================== # edX Microfrontends # - # TODO: Instead of having an nginx container for every single microfrontend - # (there are like 12 now, right?), we should come up with an actual strategy - # for micro-frontends in devtack. + # TODO: Instead of running a separate dev-server container for every single + # microfrontend (15 and counting), we probably should come up with an + # alternative strategy that is more memory efficient or at least leverages a + # shared node package cache. # ========================================================================== frontend-app-account: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-account' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-account" environment: + HEALTHCHECK_TARGET: "http://localhost:1997/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1017,15 +1079,18 @@ services: ports: - "1997:1997" depends_on: - - lms + lms: + condition: service_started designer: + extends: + file: common.yml + service: backend-app image: edxops/designer-dev:latest container_name: edx.devstack.designer hostname: designer.devstack.edx volumes: - .:/edx/app/designer - command: bash -c 'while true; do python /edx/app/designer/manage.py runserver 0.0.0.0:18808; sleep 2; done' ports: - "18808:18808" depends_on: @@ -1035,9 +1100,9 @@ services: default: aliases: - edx.devstack.designer - stdin_open: true - tty: true environment: + DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18808" + HEALTHCHECK_TARGET: "http://localhost:18808/health/" DJANGO_SETTINGS_MODULE: designer.settings.devstack ENABLE_DJANGO_TOOLBAR: 1 DB_HOST: "edx.devstack.mysql80" @@ -1048,11 +1113,12 @@ services: frontend-app-profile: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-profile' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-profile" environment: + HEALTHCHECK_TARGET: "http://localhost:1995/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1061,15 +1127,17 @@ services: ports: - "1995:1995" depends_on: - - lms + lms: + condition: service_started frontend-app-authn: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-authn' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-authn" environment: + HEALTHCHECK_TARGET: "http://localhost:1999/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1078,15 +1146,17 @@ services: ports: - "1999:1999" depends_on: - - lms + lms: + condition: service_started frontend-app-course-authoring: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-course-authoring' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-course-authoring" environment: + HEALTHCHECK_TARGET: "http://localhost:2001/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1095,11 +1165,14 @@ services: ports: - "2001:2001" depends_on: - - cms + cms: + condition: service_started frontend-app-gradebook: + environment: + HEALTHCHECK_TARGET: "http://localhost:1994/" extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-gradebook' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-gradebook" @@ -1110,15 +1183,17 @@ services: ports: - "1994:1994" depends_on: - - lms + lms: + condition: service_started frontend-app-ora-grading: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-ora-grading' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-ora-grading" environment: + HEALTHCHECK_TARGET: "http://localhost:1993/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1127,15 +1202,17 @@ services: ports: - "1993:1993" depends_on: - - lms + lms: + condition: service_started frontend-app-learner-dashboard: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-learner-dashboard' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learner-dashboard" environment: + HEALTHCHECK_TARGET: "http://localhost:1996/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1144,15 +1221,17 @@ services: ports: - "1996:1996" depends_on: - - lms + lms: + condition: service_started frontend-app-learner-record: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-learner-record' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learner-record" environment: + HEALTHCHECK_TARGET: "http://localhost:1990/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1161,15 +1240,17 @@ services: ports: - "1990:1990" depends_on: - - lms + lms: + condition: service_started frontend-app-learning: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-learning' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learning" environment: + HEALTHCHECK_TARGET: "http://localhost:${LEARNING_MICROFRONTEND_PORT:-2010}/" PARAGON_BRAND_PACKAGE: '@edx/brand-edx.org@2.x' networks: default: @@ -1178,15 +1259,17 @@ services: ports: - "${LEARNING_MICROFRONTEND_PORT:-2010}:${LEARNING_MICROFRONTEND_PORT:-2010}" depends_on: - - lms + lms: + condition: service_started frontend-app-payment: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-payment' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-payment" environment: + HEALTHCHECK_TARGET: "http://localhost:1998/" PARAGON_BRAND_PACKAGE: '@edx/brand-edx.org@^2.0.6' networks: default: @@ -1195,15 +1278,17 @@ services: ports: - "1998:1998" depends_on: - - ecommerce + ecommerce: + condition: service_started frontend-app-program-console: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-program-console' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-program-console" environment: + HEALTHCHECK_TARGET: "http://localhost:1976/" PARAGON_BRAND_PACKAGE: '@edx/brand-edx.org@1.x' networks: default: @@ -1212,16 +1297,19 @@ services: ports: - "1976:1976" depends_on: - - lms - - registrar + lms: + condition: service_started + registrar: + condition: service_started frontend-app-publisher: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-publisher' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-publisher" environment: + HEALTHCHECK_TARGET: "http://localhost:18400/" PARAGON_BRAND_PACKAGE: '@edx/elm-theme@1.x' networks: default: @@ -1230,15 +1318,17 @@ services: ports: - "18400:18400" depends_on: - - lms + lms: + condition: service_started frontend-app-admin-portal: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-admin-portal' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-admin-portal" environment: + HEALTHCHECK_TARGET: "http://localhost:1991/" PARAGON_BRAND_PACKAGE: '@edx/brand-edx.org@2.x' networks: default: @@ -1247,16 +1337,19 @@ services: ports: - "1991:1991" depends_on: - - lms - - enterprise-access + lms: + condition: service_started + enterprise-access: + condition: service_started frontend-app-learner-portal-enterprise: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-learner-portal-enterprise' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learner-portal-enterprise" environment: + HEALTHCHECK_TARGET: "http://localhost:8734/" PARAGON_BRAND_PACKAGE: '@edx/brand-edx.org@2.x' networks: default: @@ -1265,16 +1358,19 @@ services: ports: - "8734:8734" depends_on: - - lms - - enterprise-access + lms: + condition: service_started + enterprise-access: + condition: service_started frontend-app-enterprise-checkout: extends: - file: microfrontend.yml + file: common.yml service: microfrontend working_dir: '/edx/app/frontend-app-enterprise-checkout' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-enterprise-checkout" environment: + HEALTHCHECK_TARGET: "http://localhost:1989/" PARAGON_BRAND_PACKAGE: '@edx/brand-edx.org@2.x' networks: default: @@ -1283,8 +1379,10 @@ services: ports: - "1989:1989" depends_on: - - lms - - enterprise-access + lms: + condition: service_started + enterprise-access: + condition: service_started volumes: coursegraph_data: diff --git a/docs/devstack_interface.rst b/docs/devstack_interface.rst index 4a4648f4..065e15a5 100644 --- a/docs/devstack_interface.rst +++ b/docs/devstack_interface.rst @@ -47,7 +47,7 @@ Useful Commands and Summary When to use: If you only want to update one image and do not mind if the other images are behind latest. -- ``dev.up.`` - Create and start containers. i.e. brings up the container and its dependencies +- ``dev.up.`` - Create and start containers for and all dependencies, and wait until they are fully ready to serve traffic. When to use: When you are working on a specific service, use this command to bring up the necessary containers for your service i.e if working in lms, use ``make dev.up.lms`` to bring up containers for lms and its dependencies. @@ -63,7 +63,9 @@ Useful Commands and Summary + ``make dev.up.+`` will bring up , , and their dependencies - + ``make dev.up.without-deps.`` will only bring up the container + + ``make dev.up.without-deps.`` will only bring up the container, but none of the dependencies + + + ``make dev.up.without-wait.`` will only start the containers for and dependencies, but not wait for healthy status - ``dev.stop``: Stops all running containers. This does not remove the containers or the networks they had created @@ -135,7 +137,7 @@ Useful Commands and Summary Make Help ~~~~~~~~~ -The following ``make help`` output was generated in 09-2023 to make these commands searchable in documentation. +The following ``make help`` output was generated on 2026-07-31 to make these commands searchable in documentation. If you want to ensure you are getting the latest listing, simply use ``make help``. @@ -154,7 +156,6 @@ If you want to ensure you are getting the latest listing, simply use ``make help dev.clone Clone service repos to the parent directory. dev.clone.https Clone service repos using HTTPS method to the parent directory. dev.clone.ssh Clone service repos using SSH method to the parent directory. - dev.dbcopy8.% Copy data from old mysql 5.7 container into a new 8 db dev.dbshell.% Run a SQL shell on the given database. dev.destroy Irreversibly remove all devstack-related containers and networks (though not data volumes) dev.destroy.coursegraph Remove all coursegraph data. @@ -166,6 +167,7 @@ If you want to ensure you are getting the latest listing, simply use ``make help dev.logs View logs from running containers. dev.logs.% View the logs of the specified service container. dev.migrate Run migrations for applicable default services. + dev.migrate-repo-git-to-edx.% Migrate enterprise repository clones from openedx to edx GitHub org. dev.migrate.% Run migrations on a service. dev.print-container.% Get the ID of the running container for a given service. dev.provision Provision dev environment with default services, and then stop them. @@ -184,12 +186,14 @@ If you want to ensure you are getting the latest listing, simply use ``make help dev.restart-devserver.% Kill an edX service's development server. Watcher should restart it. dev.restore Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA! dev.rm-stopped Remove stopped containers. Does not affect running containers. + dev.setup-remotes Set up edx and openedx remotes for all forked repositories. dev.shell.% Run a shell on the specified service's container. dev.static.% Rebuild static assets for the specified service's container. dev.stats Get per-container CPU and memory utilization data. dev.status Prints the status of all git repositories. dev.stop Stop all running services. dev.stop.% Stop specific services. + dev.up.% Bring up services and their dependencies. dev.up.attach.% Bring up a service and its dependencies + and attach to it. dev.up.large-and-slow Bring up default services. dev.up.shell.% Bring up a service and its dependencies + shell into it. @@ -199,23 +203,12 @@ If you want to ensure you are getting the latest listing, simply use ``make help dev.up.with-watchers.% Bring up services and their dependencies + asset watcher containers. dev.up.without-deps.% Bring up services by themselves. dev.up.without-deps.shell.% Bring up a service by itself + shell into it. + dev.up.without-wait.% Bring up services and their dependencies without waiting for them to become healthy. dev.validate Print effective Docker Compose config, validating files in COMPOSE_FILE. dev.wait-for.% Wait for these services to become ready - devpi-password Get the root devpi password for the devpi container. docs generate Sphinx HTML documentation, including API docs - hadoop-application-logs-% View hadoop logs by application Id. help Display this help message. - impl-dev.clone.https Clone service repos using HTTPS method to the parent directory. - impl-dev.clone.ssh Clone service repos using SSH method to the parent directory. - impl-dev.provision Provision dev environment with default services, and then stop them. - impl-dev.provision.% Provision specified services. - impl-dev.pull.% Pull latest Docker images for services and their dependencies. - impl-dev.pull.without-deps.% Pull latest Docker images for specific services. - impl-dev.up.% Bring up services and their dependencies. - impl-dev.up.attach.% Bring up a service and its dependencies + and attach to it. - impl-dev.up.without-deps.% Bring up services by themselves. - metrics-opt-in To opt into basic data collection to help improve devstack - metrics-opt-out To opt out of metrics data collection + migrate-repo-git-to-edx Migrate enterprise repository clones from openedx to edx GitHub org. requirements install development environment requirements selfcheck Check that the Makefile is free of Make syntax errors. upgrade Upgrade requirements with pip-tools. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 67d2c982..e18f8f72 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -110,9 +110,11 @@ The default devstack services can be run by following the steps below. #. Start the desired services. This command will mount the repositories under the - ``DEVSTACK_WORKSPACE`` directory. + ``DEVSTACK_WORKSPACE`` directory into each container. - **NOTE:** it may take up to 60 seconds for the LMS to start, even after the ``dev.up.*`` command outputs ``done``. + **NOTE:** This command may take 5+ minutes because it waits for all services + to become healthy. Backend services will come online sooner than frontends, + within 1-2 minutes. Default: diff --git a/microfrontend.yml b/microfrontend.yml deleted file mode 100644 index cd3c889d..00000000 --- a/microfrontend.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This file contains configuration common to all microfrontends - -version: "2.1" - -services: - microfrontend: - # Use `npm ci` rather than `npm install` for a few reasons: - # - # - Repeatability: Respect the currently checked out package - # versions rather than upgrading when package.json and - # package-lock.json don't match. (Two people using this at - # different times on the same commit should get the same - # results.) - # - Immutability: Don't change the repo's working directory - # unexpectedly when there's a lock mismatch. - # - # Fail fast if package install fails to avoid mysterious - # errors later. - command: - - bash - - -c - - | - npm ci || exit 1 - if [ -n "$(printenv PARAGON_BRAND_PACKAGE)" ]; then - npx paragon install-theme "$(printenv PARAGON_BRAND_PACKAGE)" || exit 1 - fi - while true; do - npm start - sleep 2 - done - stdin_open: true - tty: true - image: node:18 - environment: - - NODE_ENV=development diff --git a/options.mk b/options.mk index 2085541f..d3287e0d 100644 --- a/options.mk +++ b/options.mk @@ -90,4 +90,4 @@ credentials+cms+discovery+ecommerce+insights+lms+registrar # All third-party services. # Separated by plus signs. Listed in alphabetical order for clarity. THIRD_PARTY_SERVICES ?= \ -coursegraph+elasticsearch710+memcached+mongo+mysql80+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica +coursegraph+elasticsearch710+memcached+mongo+mysql80+opensearch12+redis diff --git a/provision-coursegraph.sh b/provision-coursegraph.sh index 0652c60c..19898b8a 100755 --- a/provision-coursegraph.sh +++ b/provision-coursegraph.sh @@ -13,7 +13,7 @@ docker compose rm --force --stop coursegraph docker compose pull coursegraph echo -e "${GREEN} Starting Coursegraph and CMS...${NC}" -docker compose up -d coursegraph cms +docker compose up -d --no-deps coursegraph cms sleep 10 # Give Neo4j some time to boot up. echo -e "${GREEN} Updating CMS courses in Coursegraph...${NC}" diff --git a/provision-edx-exams.sh b/provision-edx-exams.sh index efb26bb7..1d698f5a 100755 --- a/provision-edx-exams.sh +++ b/provision-edx-exams.sh @@ -7,8 +7,7 @@ set -x name="edx-exams" port="18740" -docker compose up -d lms -docker compose up -d ${name} +docker compose up -d ${name} lms # Install requirements echo -e "${GREEN}Installing requirements for ${name}...${NC}" diff --git a/provision-enterprise-subsidy.sh b/provision-enterprise-subsidy.sh index 8cbbdd60..e1f52d3a 100755 --- a/provision-enterprise-subsidy.sh +++ b/provision-enterprise-subsidy.sh @@ -7,8 +7,7 @@ set -x name="enterprise-subsidy" port="18280" -docker compose up -d lms -docker compose up -d ${name} +docker compose up -d ${name} lms # Run migrations echo -e "${GREEN}Running migrations for ${name}...${NC}" diff --git a/provision-ida.sh b/provision-ida.sh index 2236c65d..cb008d60 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -9,7 +9,9 @@ client_name=$2 # The name of the Oauth client stored in the edxapp DB. client_port=$3 # The port corresponding to this IDA service in devstack. container_name=${4:-$1} # (Optional) The name of the container. If missing, will use app_name. -make dev.up.$app_name +# Launch the IDA service without --wait because generally the app cannot start +# successfully before migrations are run for the first time. +docker compose up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" diff --git a/provision-license-manager.sh b/provision-license-manager.sh index 938b0d5d..2956964a 100755 --- a/provision-license-manager.sh +++ b/provision-license-manager.sh @@ -8,8 +8,7 @@ set -x name=license-manager port=18170 -docker compose up -d $name -docker compose up -d lms +docker compose up -d $name lms echo -e "${GREEN}Installing requirements for ${name}...${NC}" docker compose exec -T ${name} bash -e -c 'cd /edx/app/license_manager/ && make requirements' -- "$name" diff --git a/provision-lms.sh b/provision-lms.sh index 625b8312..235e5f42 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -12,7 +12,7 @@ cms_port=18010 # Bring edxapp containers online for app in "${apps[@]}"; do - docker compose up -d $app + docker compose up -d --wait --no-deps $app done # install git for both LMS and CMS diff --git a/repo.sh b/repo.sh index dfc9dec4..96afafcb 100755 --- a/repo.sh +++ b/repo.sh @@ -21,7 +21,7 @@ fi # (or non_release_repos and non_release_ssh_repos if they are not part # of Open edX releases). repos=( - "https://github.com/openedx/course-discovery.git" + "https://github.com/edx/course-discovery.git" "https://github.com/openedx/credentials.git" "https://github.com/openedx/cs_comments_service.git" "https://github.com/edx/ecommerce.git" @@ -71,7 +71,7 @@ non_release_repos=( ) ssh_repos=( - "git@github.com:openedx/course-discovery.git" + "git@github.com:edx/course-discovery.git" "git@github.com:openedx/credentials.git" "git@github.com:openedx/cs_comments_service.git" "git@github.com:edx/ecommerce.git"