feat: update event_bus_kafka and add event_bus_redis

openedx · May 15, 2023 · d319655 · d319655
1 parent ac03146
commit d319655
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 11 deletions.
diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py
@@ -296,10 +296,10 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 #   in the LMS and CMS.
 # .. toggle_tickets: 'https://github.com/openedx/edx-platform/pull/31813'
 FEATURES['ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS'] = True
-EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
-EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
-EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
+EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:password@edx.devstack.redis:6379/'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
+EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
 
 ################# New settings must go ABOVE this line #################
 ########################################################################

diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py
@@ -496,10 +496,10 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 CLOSEST_CLIENT_IP_FROM_HEADERS = []
 
 #################### Event bus backend ########################
-EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
-EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
-EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
+EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:password@edx.devstack.redis:6379/'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
+EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
 
 ######################## Subscriptions API SETTINGS ########################
 SUBSCRIPTIONS_ROOT_URL = "http://host.docker.internal:18750"

diff --git a/requirements/edx/base.in b/requirements/edx/base.in
@@ -85,8 +85,9 @@ edx-codejail
 edx-django-utils>=5.4.0             # Utilities for cache, monitoring, and plugins
 edx-drf-extensions
 edx-enterprise
-# edx-event-bus-kafka 2.0.0 adds support for configurable producer API
-edx-event-bus-kafka>=2.0.0          # Kafka implementation of event bus
+# edx-event-bus-kafka 4.0.0 adds support for configurable consumer API
+edx-event-bus-kafka>=4.0.1          # Kafka implementation of event bus
+edx-event-bus-redis
 edx-milestones
 edx-name-affirmation
 edx-opaque-keys

diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
@@ -216,6 +216,7 @@ django==3.2.19
     #   edx-drf-extensions
     #   edx-enterprise
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-i18n-tools
     #   edx-milestones
     #   edx-name-affirmation
@@ -459,6 +460,7 @@ edx-django-utils==5.4.0
     #   edx-drf-extensions
     #   edx-enterprise
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-name-affirmation
     #   edx-rest-api-client
     #   edx-toggles
@@ -488,6 +490,8 @@ edx-enterprise==3.62.7
     #   learner-pathway-progress
 edx-event-bus-kafka==4.0.1
     # via -r requirements/edx/base.in
+edx-event-bus-redis==0.1.1
+    # via -r requirements/edx/base.in
 edx-i18n-tools==0.9.2
     # via ora2
 edx-milestones==0.4.0
@@ -543,6 +547,7 @@ edx-toggles==5.0.0
     #   -r requirements/edx/base.in
     #   edx-completion
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-name-affirmation
     #   edx-search
     #   edxval
@@ -775,6 +780,7 @@ openedx-events==7.2.0
     # via
     #   -r requirements/edx/base.in
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
 openedx-filters==1.2.0
     # via
     #   -r requirements/edx/base.in
@@ -969,7 +975,9 @@ rapidfuzz==3.0.0
 recommender-xblock==2.0.1
     # via -r requirements/edx/base.in
 redis==4.5.5
-    # via -r requirements/edx/base.in
+    # via
+    #   -r requirements/edx/base.in
+    #   walrus
 regex==2023.5.5
     # via nltk
 requests==2.30.0
@@ -1146,6 +1154,8 @@ vine==5.0.0
     #   kombu
 voluptuous==0.13.1
     # via ora2
+walrus==0.9.2
+    # via edx-event-bus-redis
 watchdog==3.0.0
     # via -r requirements/edx/paver.txt
 wcwidth==0.2.6

diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
@@ -313,6 +313,7 @@ django==3.2.19
     #   edx-drf-extensions
     #   edx-enterprise
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-i18n-tools
     #   edx-milestones
     #   edx-name-affirmation
@@ -583,6 +584,7 @@ edx-django-utils==5.4.0
     #   edx-drf-extensions
     #   edx-enterprise
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-name-affirmation
     #   edx-rest-api-client
     #   edx-toggles
@@ -612,6 +614,8 @@ edx-enterprise==3.62.7
     #   learner-pathway-progress
 edx-event-bus-kafka==4.0.1
     # via -r requirements/edx/testing.txt
+edx-event-bus-redis==0.1.1
+    # via -r requirements/edx/testing.txt
 edx-i18n-tools==0.9.2
     # via
     #   -r requirements/edx/testing.txt
@@ -676,6 +680,7 @@ edx-toggles==5.0.0
     #   -r requirements/edx/testing.txt
     #   edx-completion
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-name-affirmation
     #   edx-search
     #   edxval
@@ -1033,6 +1038,7 @@ openedx-events==7.2.0
     # via
     #   -r requirements/edx/testing.txt
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
 openedx-filters==1.2.0
     # via
     #   -r requirements/edx/testing.txt
@@ -1357,7 +1363,9 @@ rapidfuzz==3.0.0
 recommender-xblock==2.0.1
     # via -r requirements/edx/testing.txt
 redis==4.5.5
-    # via -r requirements/edx/testing.txt
+    # via
+    #   -r requirements/edx/testing.txt
+    #   walrus
 regex==2023.5.5
     # via
     #   -r requirements/edx/testing.txt
@@ -1665,6 +1673,10 @@ voluptuous==0.13.1
     #   ora2
 vulture==2.7
     # via -r requirements/edx/development.in
+walrus==0.9.2
+    # via
+    #   -r requirements/edx/testing.txt
+    #   edx-event-bus-redis
 watchdog==3.0.0
     # via -r requirements/edx/testing.txt
 wcwidth==0.2.6

diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
@@ -295,6 +295,7 @@ django==3.2.19
     #   edx-drf-extensions
     #   edx-enterprise
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-i18n-tools
     #   edx-milestones
     #   edx-name-affirmation
@@ -561,6 +562,7 @@ edx-django-utils==5.4.0
     #   edx-drf-extensions
     #   edx-enterprise
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-name-affirmation
     #   edx-rest-api-client
     #   edx-toggles
@@ -590,6 +592,8 @@ edx-enterprise==3.62.7
     #   learner-pathway-progress
 edx-event-bus-kafka==4.0.1
     # via -r requirements/edx/base.txt
+edx-event-bus-redis==0.1.1
+    # via -r requirements/edx/base.txt
 edx-i18n-tools==0.9.2
     # via
     #   -r requirements/edx/base.txt
@@ -653,6 +657,7 @@ edx-toggles==5.0.0
     #   -r requirements/edx/base.txt
     #   edx-completion
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
     #   edx-name-affirmation
     #   edx-search
     #   edxval
@@ -980,6 +985,7 @@ openedx-events==7.2.0
     # via
     #   -r requirements/edx/base.txt
     #   edx-event-bus-kafka
+    #   edx-event-bus-redis
 openedx-filters==1.2.0
     # via
     #   -r requirements/edx/base.txt
@@ -1279,7 +1285,9 @@ rapidfuzz==3.0.0
 recommender-xblock==2.0.1
     # via -r requirements/edx/base.txt
 redis==4.5.5
-    # via -r requirements/edx/base.txt
+    # via
+    #   -r requirements/edx/base.txt
+    #   walrus
 regex==2023.5.5
     # via
     #   -r requirements/edx/base.txt
@@ -1537,6 +1545,10 @@ voluptuous==0.13.1
     # via
     #   -r requirements/edx/base.txt
     #   ora2
+walrus==0.9.2
+    # via
+    #   -r requirements/edx/base.txt
+    #   edx-event-bus-redis
 watchdog==3.0.0
     # via -r requirements/edx/base.txt
 wcwidth==0.2.6