diff --git a/Makefile b/Makefile index 62cdff8b79e6..51505257759e 100644 --- a/Makefile +++ b/Makefile @@ -66,11 +66,17 @@ lint: poetry run black --check rasa tests make lint-docstrings -BRANCH ?= master # Compare against `master` if no branch was provided +# Compare against `master` if no branch was provided +BRANCH ?= "master" lint-docstrings: - # Lint docstrings only against the the diff to avoid too many errors. - # Check only production code. Ignore other flake errors which are captured by `lint` - git diff $(BRANCH) -- rasa | poetry run flake8 --select D --diff + # Lint docstrings only against the the diff to avoid too many errors. + # Check only production code. Ignore other flake errors which are captured by `lint` + # Diff of committed changes (shows only changes introduced by your branch) + if [[ -n "$(BRANCH)" ]]; then \ + git diff $(BRANCH)...HEAD -- rasa | poetry run flake8 --select D --diff; \ + fi + # Diff of uncommitted changes for running locally + git diff HEAD -- rasa | poetry run flake8 --select D --diff types: # FIXME: working our way towards removing these diff --git a/changelog/6498.improvement.md b/changelog/6498.improvement.md new file mode 100644 index 000000000000..7bf8dc188ecd --- /dev/null +++ b/changelog/6498.improvement.md @@ -0,0 +1,2 @@ +Adding configurable prefixes to Redis [Tracker](./tracker-stores.mdx) and [Lock Stores](./lock-stores.mdx) so that a single Redis instance (and logical DB) can support multiple conversation trackers and locks. +By default, conversations will be prefixed with `tracker:...` and all locks prefixed with `lock:...`. Additionally, you can add an alphanumeric-only `prefix: value` in `endpoints.yml` such that keys in redis will take the form `value:tracker:...` and `value:lock:...` respectively. diff --git a/data/test_endpoints/example_endpoints.yml b/data/test_endpoints/example_endpoints.yml index 455e5d58b2f9..436cd1a15344 100644 --- a/data/test_endpoints/example_endpoints.yml +++ b/data/test_endpoints/example_endpoints.yml @@ -14,6 +14,7 @@ tracker_store: port: 6379 db: 0 password: password + key_prefix: conversation record_exp: 30000 # example of mongoDB external tracker store config #tracker_store: diff --git a/docs/docs/components.mdx b/docs/docs/components.mdx index 4bdd3e6cd220..c3cf7a003e24 100644 --- a/docs/docs/components.mdx +++ b/docs/docs/components.mdx @@ -2714,11 +2714,6 @@ See [starspace paper](https://arxiv.org/abs/1709.03856) for details. You can create a custom component to perform a specific task which NLU doesn't currently offer (for example, sentiment analysis). Below is the specification of the `rasa.nlu.components.Component`] class with the methods you'll need to implement. -:::tip follow the tutorial -There is a detailed tutorial on building custom components on the [Rasa Blog](https://blog.rasa.com/enhancing-rasa-nlu-with-custom-components/). - -::: - You can add a custom component to your pipeline by adding the module path. So if you have a module called `sentiment` containing a `SentimentAnalyzer` class: diff --git a/docs/docs/event-brokers.mdx b/docs/docs/event-brokers.mdx index 1cb1b51474e3..ddc3c10bc9b1 100644 --- a/docs/docs/event-brokers.mdx +++ b/docs/docs/event-brokers.mdx @@ -123,19 +123,22 @@ If using the `SSL` protocol, the endpoints file should look like: ### Adding a Kafka Broker in Python -The code below shows an example on how to instantiate a Kafka producer in you script. +The code below shows an example on how to instantiate a Kafka producer in your script. ```python from rasa.core.brokers.kafka import KafkaEventBroker from rasa.core.tracker_store import InMemoryTrackerStore +from rasa.shared.core.domain import Domain -kafka_broker = KafkaEventBroker(host='localhost:9092', +kafka_broker = KafkaEventBroker(url='localhost:9092', topic='rasa_events') + +domain = Domain.load("domain.yml") tracker_store = InMemoryTrackerStore(domain=domain, event_broker=kafka_broker) ``` -The host variable can be either a list of brokers addresses or a single one. +The `url` argument can be either a list of broker addresses or a single one. If only one broker address is available, the client will connect to it and request the cluster Metadata. Therefore, the remaining brokers in the cluster can be discovered diff --git a/docs/docs/lock-stores.mdx b/docs/docs/lock-stores.mdx index c488e46c463b..4faa59fc0230 100644 --- a/docs/docs/lock-stores.mdx +++ b/docs/docs/lock-stores.mdx @@ -57,6 +57,7 @@ address the same node when sending messages for a given conversation ID. port: password: db: + key_prefix: ``` 3. To start the Rasa Core server using your Redis backend, add the `--endpoints` @@ -76,10 +77,13 @@ address the same node when sending messages for a given conversation ID. * `db` (default: `1`): The number of your redis database + * `key_prefix` (default: `None`): The prefix to prepend to lock store keys. Must + be alphanumeric + * `password` (default: `None`): Password used for authentication (`None` equals no authentication) * `use_ssl` (default: `False`): Whether or not the communication is encrypted * `socket_timeout` (default: `10`): Time in seconds after which an - error is raised if Redis doesn't answer \ No newline at end of file + error is raised if Redis doesn't answer diff --git a/docs/docs/tracker-stores.mdx b/docs/docs/tracker-stores.mdx index b2c7448e3edd..eeb5f350a6f0 100644 --- a/docs/docs/tracker-stores.mdx +++ b/docs/docs/tracker-stores.mdx @@ -179,14 +179,12 @@ To set up Rasa Open Source with Redis the following steps are required: type: redis url: port: + key_prefix: db: password: use_ssl: ``` -3. To start the Rasa server using your configured Redis instance, - add the `--endpoints` flag, e.g.: - ```bash rasa run -m models --endpoints endpoints.yml ``` @@ -207,18 +205,20 @@ To set up Rasa Open Source with Redis the following steps are required: url: port: db: + key_prefix: password: use_ssl: ``` -#### Configuration Parameters - * `url` (default: `localhost`): The url of your redis instance * `port` (default: `6379`): The port which redis is running on * `db` (default: `0`): The number of your redis database +* `key_prefix` (default: `None`): The prefix to prepend to tracker store keys. Must + be alphanumeric + * `password` (default: `None`): Password used for authentication (`None` equals no authentication) diff --git a/docs/docs/training-data-format.mdx b/docs/docs/training-data-format.mdx index 6c5fdd42e149..887c269b4de6 100644 --- a/docs/docs/training-data-format.mdx +++ b/docs/docs/training-data-format.mdx @@ -3,7 +3,7 @@ id: training-data-format sidebar_label: Training Data Format title: Training Data Format description: Description of the YAML format for training data -abstract: This page provides an overview of the different types of training data +abstract: This page describes the different types of training data that go into a Rasa assistant and how this training data is structured. --- @@ -13,10 +13,9 @@ Rasa Open Source uses [YAML](https://yaml.org/spec/1.2/spec.html) as a unified and extendable way to manage all training data, including NLU data, stories and rules. -With the YAML format, training data can be split over any number of YAML files, -and every file can contain any kind of data. -The training data parser will read the top level keys in each file to decide -what kind of data is in a section at training time. +You can split the training data over any number of YAML files, +and each file can contain any combination of NLU data, stories, and rules. +The training data parser determines the training data type using top level keys. The [domain](glossary.mdx#domain) uses the same YAML format as the training data and can also be split across @@ -32,21 +31,20 @@ you can still find the documentation for None: if hasattr(cmdline_arguments, "func"): rasa.utils.io.configure_colored_logging(log_level) set_log_and_warnings_filters() + rasa.telemetry.initialize_telemetry() rasa.telemetry.initialize_error_reporting() cmdline_arguments.func(cmdline_arguments) elif hasattr(cmdline_arguments, "version"): diff --git a/rasa/core/lock_store.py b/rasa/core/lock_store.py index e02b21b0b9f3..9a88a1214e42 100644 --- a/rasa/core/lock_store.py +++ b/rasa/core/lock_store.py @@ -22,6 +22,8 @@ def _get_lock_lifetime() -> int: LOCK_LIFETIME = _get_lock_lifetime() DEFAULT_SOCKET_TIMEOUT_IN_SECONDS = 10 +DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX = "lock:" + # noinspection PyUnresolvedReferences class LockError(RasaException): @@ -199,6 +201,7 @@ def __init__( db: int = 1, password: Optional[Text] = None, use_ssl: bool = False, + key_prefix: Optional[Text] = None, socket_timeout: float = DEFAULT_SOCKET_TIMEOUT_IN_SECONDS, ) -> None: """Create a lock store which uses Redis for persistence. @@ -211,6 +214,8 @@ def __init__( password: The password which should be used for authentication with the Redis database. use_ssl: `True` if SSL should be used for the connection to Redis. + key_prefix: prefix to prepend to all keys used by the lock store. Must be + alphanumeric. socket_timeout: Timeout in seconds after which an exception will be raised in case Redis doesn't respond within `socket_timeout` seconds. """ @@ -224,19 +229,36 @@ def __init__( ssl=use_ssl, socket_timeout=socket_timeout, ) + + self.key_prefix = DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX + if key_prefix: + logger.debug(f"Setting non-default redis key prefix: '{key_prefix}'.") + self._set_key_prefix(key_prefix) + super().__init__() + def _set_key_prefix(self, key_prefix: Text) -> None: + if isinstance(key_prefix, str) and key_prefix.isalnum(): + self.key_prefix = key_prefix + ":" + DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX + else: + logger.warning( + f"Omitting provided non-alphanumeric redis key prefix: '{key_prefix}'. Using default '{self.key_prefix}' instead." + ) + + def _get_key_prefix(self) -> Text: + return self.key_prefix + def get_lock(self, conversation_id: Text) -> Optional[TicketLock]: - serialised_lock = self.red.get(conversation_id) + serialised_lock = self.red.get(self.key_prefix + conversation_id) if serialised_lock: return TicketLock.from_dict(json.loads(serialised_lock)) def delete_lock(self, conversation_id: Text) -> None: - deletion_successful = self.red.delete(conversation_id) + deletion_successful = self.red.delete(self.key_prefix + conversation_id) self._log_deletion(conversation_id, deletion_successful) def save_lock(self, lock: TicketLock) -> None: - self.red.set(lock.conversation_id, lock.dumps()) + self.red.set(self.key_prefix + lock.conversation_id, lock.dumps()) class InMemoryLockStore(LockStore): diff --git a/rasa/core/tracker_store.py b/rasa/core/tracker_store.py index 9dfcf745b43c..81a4fa670a8b 100644 --- a/rasa/core/tracker_store.py +++ b/rasa/core/tracker_store.py @@ -58,6 +58,9 @@ POSTGRESQL_DEFAULT_MAX_OVERFLOW = 100 POSTGRESQL_DEFAULT_POOL_SIZE = 50 +# default value for key prefix in RedisTrackerStore +DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX = "tracker:" + class TrackerStore: """Class to hold all of the TrackerStore classes""" @@ -305,6 +308,7 @@ def __init__( password: Optional[Text] = None, event_broker: Optional[EventBroker] = None, record_exp: Optional[float] = None, + key_prefix: Optional[Text] = None, use_ssl: bool = False, **kwargs: Dict[Text, Any], ) -> None: @@ -314,8 +318,25 @@ def __init__( host=host, port=port, db=db, password=password, ssl=use_ssl ) self.record_exp = record_exp + + self.key_prefix = DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX + if key_prefix: + logger.debug(f"Setting non-default redis key prefix: '{key_prefix}'.") + self._set_key_prefix(key_prefix) + super().__init__(domain, event_broker, **kwargs) + def _set_key_prefix(self, key_prefix: Text) -> None: + if isinstance(key_prefix, str) and key_prefix.isalnum(): + self.key_prefix = key_prefix + ":" + DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX + else: + logger.warning( + f"Omitting provided non-alphanumeric redis key prefix: '{key_prefix}'. Using default '{self.key_prefix}' instead." + ) + + def _get_key_prefix(self) -> Text: + return self.key_prefix + def save(self, tracker, timeout=None): """Saves the current conversation state""" if self.event_broker: @@ -325,10 +346,10 @@ def save(self, tracker, timeout=None): timeout = self.record_exp serialised_tracker = self.serialise_tracker(tracker) - self.red.set(tracker.sender_id, serialised_tracker, ex=timeout) + self.red.set(self.prefix + tracker.sender_id, serialised_tracker, ex=timeout) def retrieve(self, sender_id: Text) -> Optional[DialogueStateTracker]: - stored = self.red.get(sender_id) + stored = self.red.get(self.prefix + sender_id) if stored is not None: return self.deserialise_tracker(sender_id, stored) else: @@ -336,7 +357,7 @@ def retrieve(self, sender_id: Text) -> Optional[DialogueStateTracker]: def keys(self) -> Iterable[Text]: """Returns keys of the Redis Tracker Store""" - return self.red.keys() + return self.red.keys(self.prefix + "*") class DynamoTrackerStore(TrackerStore): diff --git a/rasa/telemetry.py b/rasa/telemetry.py index d0444b5d33a7..c3573ff51702 100644 --- a/rasa/telemetry.py +++ b/rasa/telemetry.py @@ -126,8 +126,11 @@ def _write_default_telemetry_configuration( CONFIG_FILE_TELEMETRY_KEY, new_config ) - if is_enabled and success: + # Do not show info if user has enabled/disabled telemetry via env var + telemetry_environ = os.environ.get(TELEMETRY_ENABLED_ENVIRONMENT_VARIABLE) + if is_enabled and success and telemetry_environ is None: print_telemetry_reporting_info() + return success @@ -148,7 +151,7 @@ def _is_telemetry_enabled_in_configuration() -> bool: except ValueError as e: logger.debug(f"Could not read telemetry settings from configuration file: {e}") - # seems like there is no config, we'll create on and enable telemetry + # seems like there is no config, we'll create one and enable telemetry success = _write_default_telemetry_configuration() # if writing the configuration failed, telemetry will be disabled return TELEMETRY_ENABLED_BY_DEFAULT and success @@ -162,16 +165,16 @@ def is_telemetry_enabled() -> bool: """ telemetry_environ = os.environ.get(TELEMETRY_ENABLED_ENVIRONMENT_VARIABLE) - if telemetry_environ is None: - try: - return rasa_utils.read_global_config_value( - CONFIG_FILE_TELEMETRY_KEY, unavailable_ok=False - )[CONFIG_TELEMETRY_ENABLED] - except ValueError: - return False - else: + if telemetry_environ is not None: return telemetry_environ.lower() == "true" + try: + return rasa_utils.read_global_config_value( + CONFIG_FILE_TELEMETRY_KEY, unavailable_ok=False + )[CONFIG_TELEMETRY_ENABLED] + except ValueError: + return False + def initialize_telemetry() -> bool: """Read telemetry configuration from the user's Rasa config file in $HOME. @@ -190,8 +193,8 @@ def initialize_telemetry() -> bool: if telemetry_environ is None: return is_enabled_in_configuration - else: - return telemetry_environ.lower() == "true" + + return telemetry_environ.lower() == "true" except Exception as e: # skipcq:PYL-W0703 logger.exception( f"Failed to initialize telemetry reporting: {e}." @@ -210,10 +213,6 @@ def ensure_telemetry_enabled(f: Callable[..., Any]) -> Callable[..., Any]: Returns: Return wrapped function """ - # checks if telemetry is enabled and creates a default config if this is the first - # call to it - initialize_telemetry() - # allows us to use the decorator for async and non async functions if asyncio.iscoroutinefunction(f): @@ -224,15 +223,14 @@ async def decorated(*args, **kwargs): return None return decorated - else: - @wraps(f) - def decorated(*args, **kwargs): - if is_telemetry_enabled(): - return f(*args, **kwargs) - return None + @wraps(f) + def decorated(*args, **kwargs): + if is_telemetry_enabled(): + return f(*args, **kwargs) + return None - return decorated + return decorated def _fetch_write_key(tool: Text, environment_variable: Text) -> Optional[Text]: @@ -641,19 +639,21 @@ def initialize_error_reporting() -> None: environment="development" if in_continuous_integration() else "production", ) - if telemetry_id: - with configure_scope() as scope: - # sentry added these more recently, just a protection in a case where a - # user has installed an older version of sentry - if hasattr(scope, "set_user"): - scope.set_user({"id": telemetry_id}) - - default_context = _default_context_fields() - if hasattr(scope, "set_context"): - if "os" in default_context: - # os is a nested dict, hence we report it separately - scope.set_context("Operating System", default_context.pop("os")) - scope.set_context("Environment", default_context) + if not telemetry_id: + return + + with configure_scope() as scope: + # sentry added these more recently, just a protection in a case where a + # user has installed an older version of sentry + if hasattr(scope, "set_user"): + scope.set_user({"id": telemetry_id}) + + default_context = _default_context_fields() + if hasattr(scope, "set_context"): + if "os" in default_context: + # os is a nested dict, hence we report it separately + scope.set_context("Operating System", default_context.pop("os")) + scope.set_context("Environment", default_context) @async_generator.asynccontextmanager diff --git a/security.txt b/security.txt new file mode 100644 index 000000000000..f52c4b027b82 --- /dev/null +++ b/security.txt @@ -0,0 +1,4 @@ +Contact: mailto:security@rasa.com +Preferred-Languages: en +Canonical: https://github.com/RasaHQ/rasa/blob/master/security.txt +Hiring: https://rasa.com/jobs diff --git a/tests/core/test_lock_store.py b/tests/core/test_lock_store.py index ae7a5e083379..96f330c9f58d 100644 --- a/tests/core/test_lock_store.py +++ b/tests/core/test_lock_store.py @@ -16,7 +16,13 @@ from rasa.core.constants import DEFAULT_LOCK_LIFETIME from rasa.shared.constants import INTENT_MESSAGE_PREFIX from rasa.core.lock import TicketLock -from rasa.core.lock_store import InMemoryLockStore, LockError, LockStore, RedisLockStore +from rasa.core.lock_store import ( + InMemoryLockStore, + LockError, + LockStore, + RedisLockStore, + DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX, +) class FakeRedisLockStore(RedisLockStore): @@ -32,6 +38,8 @@ def __init__(self): # added in redis==3.3.0, but not yet in fakeredis self.red.connection_pool.connection_class.health_check_interval = 0 + self.key_prefix = DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX + def test_issue_ticket(): lock = TicketLock("random id 0") @@ -287,3 +295,46 @@ async def test_redis_lock_store_timeout(monkeypatch: MonkeyPatch): with pytest.raises(LockError): async with lock_store.lock("some sender"): pass + + +async def test_redis_lock_store_with_invalid_prefix(monkeypatch: MonkeyPatch): + import redis.exceptions + + lock_store = FakeRedisLockStore() + + prefix = "!asdf234 34#" + lock_store._set_key_prefix(prefix) + assert lock_store._get_key_prefix() == DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX + + monkeypatch.setattr( + lock_store, + lock_store.get_or_create_lock.__name__, + Mock(side_effect=redis.exceptions.TimeoutError), + ) + + with pytest.raises(LockError): + async with lock_store.lock("some sender"): + pass + + +async def test_redis_lock_store_with_valid_prefix(monkeypatch: MonkeyPatch): + import redis.exceptions + + lock_store = FakeRedisLockStore() + + prefix = "chatbot42" + lock_store._set_key_prefix(prefix) + assert ( + lock_store._get_key_prefix() + == prefix + ":" + DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX + ) + + monkeypatch.setattr( + lock_store, + lock_store.get_or_create_lock.__name__, + Mock(side_effect=redis.exceptions.TimeoutError), + ) + + with pytest.raises(LockError): + async with lock_store.lock("some sender"): + pass diff --git a/tests/core/test_tracker_stores.py b/tests/core/test_tracker_stores.py index 3590080e2dd8..07defc9be0e3 100644 --- a/tests/core/test_tracker_stores.py +++ b/tests/core/test_tracker_stores.py @@ -35,6 +35,7 @@ TrackerStore, InMemoryTrackerStore, RedisTrackerStore, + DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX, SQLTrackerStore, DynamoTrackerStore, FailSafeTrackerStore, @@ -148,6 +149,42 @@ def test_create_tracker_store_from_endpoint_config(default_domain: Domain): assert isinstance(tracker_store, type(TrackerStore.create(store, default_domain))) +def test_redis_tracker_store_invalid_key_prefix(default_domain: Domain): + + test_invalid_key_prefix = "$$ &!" + + tracker_store = RedisTrackerStore( + domain=default_domain, + host="localhost", + port=6379, + db=0, + password="password", + key_prefix=test_invalid_key_prefix, + record_exp=3000, + ) + + assert tracker_store._get_key_prefix() == DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX + + +def test_redis_tracker_store_valid_key_prefix(default_domain: Domain): + test_valid_key_prefix = "spanish" + + tracker_store = RedisTrackerStore( + domain=default_domain, + host="localhost", + port=6379, + db=0, + password="password", + key_prefix=test_valid_key_prefix, + record_exp=3000, + ) + + assert ( + tracker_store._get_key_prefix() + == f"{test_valid_key_prefix}:{DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX}" + ) + + def test_exception_tracker_store_from_endpoint_config( default_domain: Domain, monkeypatch: MonkeyPatch ): diff --git a/tests/shared/core/test_trackers.py b/tests/shared/core/test_trackers.py index 6368eb4eab7d..ab68af9ec156 100644 --- a/tests/shared/core/test_trackers.py +++ b/tests/shared/core/test_trackers.py @@ -83,6 +83,9 @@ def __init__(self, _domain: Domain) -> None: # added in redis==3.3.0, but not yet in fakeredis self.red.connection_pool.connection_class.health_check_interval = 0 + # Defined in RedisTrackerStore but needs to be added for the MockRedisTrackerStore + self.prefix = "tracker:" + TrackerStore.__init__(self, _domain)