From 5e3aa66f728fd32e64e419a8f6b91e432e55027f Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Tue, 19 Nov 2024 16:36:04 +0500 Subject: [PATCH] feat: add license manager ida to devstack --- .github/workflows/provisioning-tests.yml | 2 +- check.sh | 6 ++ docker-compose-host.yml | 12 ++++ docker-compose.yml | 70 +++++++++++++++++++++++ docs/service_list.rst | 3 + options.mk | 4 +- provision-license-manager.sh | 32 +++++++++++ provision-mysql80.sql | 4 ++ provision.sh | 1 + provision.sql | 2 + py_configuration_files/license_manager.py | 10 ++-- repo.sh | 2 + 12 files changed, 140 insertions(+), 8 deletions(-) create mode 100755 provision-license-manager.sh diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index 5af1594975..c72a0b9b41 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -30,7 +30,7 @@ jobs: os: - ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa" python-version: [ '3.11' ] - services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms] + services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms, license-manager] fail-fast: false # some services can be flaky; let others run to completion even if one fails steps: diff --git a/check.sh b/check.sh index 3c0781f025..f010e39299 100755 --- a/check.sh +++ b/check.sh @@ -156,6 +156,12 @@ if should_check analyticsapi; then "curl --fail -L http://localhost:19001/health/" fi +if should_check license-manager; then + echo "Running License Manager Devstack tests: " + run_check license_manager_heartbeat license-manager \ + "curl --fail -L http://localhost:18170/health/" +fi + echo "Successful checks:${succeeded:- NONE}" echo "Failed checks:${failed:- NONE}" if [[ -z "$succeeded" ]] && [[ -z "$failed" ]]; then diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 85f7a2a272..7676345856 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -71,6 +71,18 @@ services: - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api - ${DEVSTACK_WORKSPACE}/src:/edx/src - ${PWD}/py_configuration_files/analytics_data_api.py:/edx/app/analytics_api/analytics_api/analyticsdataserver/settings/devstack.py + license-manager: + volumes: + - ${DEVSTACK_WORKSPACE}/license-manager:/edx/app/license-manager + - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached + license-manager-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/license-manager:/edx/app/license-manager + - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached + bulk-enrollment-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/license-manager:/edx/app/license-manager + - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached # Note that frontends mount `src` to /edx/app/src instead of /edx/src. # See ADR #5 for rationale. diff --git a/docker-compose.yml b/docker-compose.yml index a165a6bf22..d74989dbac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -693,6 +693,76 @@ services: aliases: - edx.devstack.xqueue_consumer + license-manager: + image: edxops/license-manager-dev:latest + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.license-manager" + hostname: license-manager.devstack.edx + # Use the Django devserver, so that we can hot-reload code changes + command: bash -c 'pip install -r /edx/app/license_manager/requirements/dev.txt && while true; do python /edx/app/license_manager/manage.py runserver 0.0.0.0:18170; sleep 2; done' + ports: + - "18170:18170" + depends_on: + - mysql80 + - license-manager-worker + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: license_manager.settings.devstack + DJANGO_WATCHMAN_TIMEOUT: 30 + ENABLE_DJANGO_TOOLBAR: 1 + VIRTUAL_ENV: /edx/app/license_manager/venvs/license_manager + + license-manager-worker: + image: edxops/license-manager-dev:latest + command: bash -c 'pip install -r /edx/app/license_manager/requirements/dev.txt && cd /edx/app/license_manager/license_manager && celery -A license_manager worker -Q license_manager.default -l DEBUG' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.license-manager-worker" + hostname: license-manager-worker.devstack.edx + depends_on: + - mysql80 + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: license_manager.settings.devstack + COLUMNS: 80 + VIRTUAL_ENV: /edx/app/license_manager/venvs/license_manager + ports: + - "18171:18171" + restart: always + stdin_open: true + tty: true + + bulk-enrollment-worker: + image: edxops/license-manager-dev:latest + command: bash -c 'pip install -r /edx/app/license_manager/requirements/dev.txt && cd /edx/app/license_manager/license_manager && celery -A license_manager worker -Q license_manager.bulk_enrollment -l DEBUG' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.license-manager.bulk-enrollment-worker" + hostname: license-manager.bulk-enrollment-worker.devstack.edx + depends_on: + - mysql80 + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: license_manager.settings.devstack + COLUMNS: 80 + VIRTUAL_ENV: /edx/app/license_manager/venvs/license_manager + ports: + - "18172:18172" + restart: always + stdin_open: true + tty: true + + # ========================================================================== # edX Microfrontends # diff --git a/docs/service_list.rst b/docs/service_list.rst index 092b8028c6..9327905fc4 100644 --- a/docs/service_list.rst +++ b/docs/service_list.rst @@ -61,6 +61,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro +------------------------------------+-------------------------------------+----------------+--------------+ | `analyticsapi`_ | http://localhost:19001 | Python/Django | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ +| `license-manager`_ | http://localhost:18171 | Python/Django | Extra | ++------------------------------------+-------------------------------------+----------------+--------------+ | `frontend-app-ora-grading`_ | http://localhost:1993 | MFE (React.js) | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ @@ -95,3 +97,4 @@ Some common service combinations include: .. _frontend-app-ora-grading: https://github.com/edx/frontend-app-ora-grading .. _insights: https://github.com/edx/edx-analytics-dashboard .. _analyticsapi: https://github.com/edx/edx-analytics-data-api +.. _license-manager: https://github.com/openedx/license-manager diff --git a/options.mk b/options.mk index 5a4d9b4e2c..8c26911118 100644 --- a/options.mk +++ b/options.mk @@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend- # Separated by plus signs. # Separated by plus signs. Listed in alphabetical order for clarity. EDX_SERVICES ?= \ -analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer +analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+license-manager+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer # Services with database migrations. # Should be a subset of $(EDX_SERVICES). @@ -76,7 +76,7 @@ analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_note # Note: This list should contain _all_ db-backed services, even if not # configured to run; the list will be filtered later against $(DEFAULT_SERVICES). DB_SERVICES ?= \ -credentials+cms+discovery+ecommerce+lms+registrar +credentials+cms+discovery+ecommerce+lms+registrar+license-manager # Services with static assets to be built. # Should be a subset of $(EDX_SERVICES). diff --git a/provision-license-manager.sh b/provision-license-manager.sh new file mode 100755 index 0000000000..d97b983b54 --- /dev/null +++ b/provision-license-manager.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Provisioning script for the notes service +set -eu -o pipefail + +. scripts/colors.sh +set -x + +name=license-manager +port=18170 + +docker compose up -d $name + +echo -e "${GREEN}Installing requirements for ${name}...${NC}" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/license_manager/ && make requirements' +# Run migrations +echo -e "${GREEN}Running migrations for ${name}...${NC}" +docker exec -t ${name} bash -c "cd /edx/app/license_manager/ && make migrate" + +# Seed data for development +echo -e "${GREEN}Seeding development data..." +docker exec -t ${name} bash -c "python manage.py seed_development_data" +# Some migrations require development data to be seeded, hence migrating again. +docker exec -t ${name} bash -c "make migrate" + +# Create superuser +echo -e "${GREEN}Creating super-user for ${name}...${NC}" +docker exec -t ${name} bash -c "echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None' | python /edx/app/license_manager/manage.py shell" + +# Provision IDA User in LMS +./provision-ida-user.sh ${name}-backend-service ${name}_worker $port + +make dev.restart-devserver.license-manager diff --git a/provision-mysql80.sql b/provision-mysql80.sql index 19b32e0ebd..461aa5551c 100644 --- a/provision-mysql80.sql +++ b/provision-mysql80.sql @@ -39,6 +39,10 @@ GRANT ALL ON `reports`.* TO 'analytics001'@'%'; CREATE DATABASE IF NOT EXISTS `reports_v1`; GRANT ALL ON `reports_v1`.* TO 'analytics001'@'%'; +CREATE DATABASE IF NOT EXISTS license_manager; +CREATE USER IF NOT EXISTS 'license_manager001'@'%' IDENTIFIED BY 'password'; +GRANT ALL ON license_manager.* TO 'license_manager001'@'%'; + CREATE DATABASE IF NOT EXISTS edxapp; CREATE DATABASE IF NOT EXISTS edxapp_csmh; CREATE USER IF NOT EXISTS 'edxapp001'@'%' IDENTIFIED BY 'password'; diff --git a/provision.sh b/provision.sh index cde27b6fcd..6aeed7d03b 100755 --- a/provision.sh +++ b/provision.sh @@ -49,6 +49,7 @@ xqueue \ coursegraph \ insights \ analyticsapi \ +license-manager \ " # What should we provision? diff --git a/provision.sql b/provision.sql index 0b672c9986..6d7090a716 100644 --- a/provision.sql +++ b/provision.sql @@ -33,5 +33,7 @@ GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; CREATE DATABASE IF NOT EXISTS `reports_v1`; GRANT ALL ON `reports_v1`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +CREATE DATABASE IF NOT EXISTS license_manager; +GRANT ALL ON license_manager.* TO 'license_manager001'@'%' IDENTIFIED BY 'password'; FLUSH PRIVILEGES; diff --git a/py_configuration_files/license_manager.py b/py_configuration_files/license_manager.py index 468e623f87..ca72fa6be8 100644 --- a/py_configuration_files/license_manager.py +++ b/py_configuration_files/license_manager.py @@ -9,11 +9,11 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'license_manager', - 'USER': 'root', - 'PASSWORD': '', - 'HOST': 'license-manager.mysql', - 'PORT': '3306', + 'NAME': os.environ.get('DB_NAME', 'license_manager'), + 'USER': os.environ.get('DB_USER', 'license_manager001'), + 'PASSWORD': os.environ.get('DB_PASSWORD', 'password'), + 'HOST': os.environ.get('DB_HOST', 'edx.devstack.mysql80'), + 'PORT': os.environ.get('DB_PORT', 3306), } } diff --git a/repo.sh b/repo.sh index cb93108f63..d3ae862955 100755 --- a/repo.sh +++ b/repo.sh @@ -36,6 +36,7 @@ repos=( "https://github.com/openedx/frontend-app-publisher.git" "https://github.com/edx/edx-analytics-dashboard.git" "https://github.com/edx/edx-analytics-data-api.git" + "https://github.com/openedx/license-manager.git" ) non_release_repos=( @@ -66,6 +67,7 @@ ssh_repos=( "git@github.com:openedx/frontend-app-publisher.git" "git@github.com:edx/edx-analytics-dashboard.git" "git@github.com:edx/edx-analytics-data-api.git" + "git@github.com:openedx/license-manager.git" ) non_release_ssh_repos=(