Merge pull request #5180 from RasaHQ/use-file-utils

use file utils with proper encoding instead of open

changelog/5180.misc.rst

@@ -0,0 +1 @@
+Improved file loading by making sure we use our file helpers everywhere possible.

tests/cli/test_rasa_x.py

@@ -148,7 +148,5 @@ async def test_pull_runtime_config_from_server():
             config_url, 1, 0
         )
 
-        with open(endpoints_path) as f:
-            assert f.read() == endpoint_config
-        with open(credentials_path) as f:
-            assert f.read() == credentials
+        assert io_utils.read_file(endpoints_path) == endpoint_config
+        assert io_utils.read_file(credentials_path) == credentials

tests/core/test_broker.py

@@ -96,36 +96,40 @@ def test_file_broker_from_config():
 
 
 def test_file_broker_logs_to_file(tmpdir):
-    fname = tmpdir.join("events.log").strpath
+    log_file_path = tmpdir.join("events.log").strpath
 
-    actual = EventBroker.create(EndpointConfig(**{"type": "file", "path": fname}))
+    actual = EventBroker.create(
+        EndpointConfig(**{"type": "file", "path": log_file_path})
+    )
 
     for e in TEST_EVENTS:
         actual.publish(e.as_dict())
 
     # reading the events from the file one event per line
     recovered = []
-    with open(fname, "r") as f:
-        for l in f:
-            recovered.append(Event.from_parameters(json.loads(l)))
+    with open(log_file_path, "r") as log_file:
+        for line in log_file:
+            recovered.append(Event.from_parameters(json.loads(line)))
 
     assert recovered == TEST_EVENTS
 
 
 def test_file_broker_properly_logs_newlines(tmpdir):
-    fname = tmpdir.join("events.log").strpath
+    log_file_path = tmpdir.join("events.log").strpath
 
-    actual = EventBroker.create(EndpointConfig(**{"type": "file", "path": fname}))
+    actual = EventBroker.create(
+        EndpointConfig(**{"type": "file", "path": log_file_path})
+    )
 
     event_with_newline = UserUttered("hello \n there")
 
     actual.publish(event_with_newline.as_dict())
 
     # reading the events from the file one event per line
     recovered = []
-    with open(fname, "r") as f:
-        for l in f:
-            recovered.append(Event.from_parameters(json.loads(l)))
+    with open(log_file_path, "r") as log_file:
+        for line in log_file:
+            recovered.append(Event.from_parameters(json.loads(line)))
 
     assert recovered == [event_with_newline]
 

tests/core/test_dsl.py

@@ -2,12 +2,14 @@
 
 import json
 from collections import Counter
+from pathlib import Path
 from typing import Text, Dict
 
 import numpy as np
 import pytest
 
-from rasa.core import training
+import rasa.utils.io
+from rasa.core import training, utils
 from rasa.core.interpreter import RegexInterpreter
 from rasa.core.training.dsl import StoryFileReader, EndToEndReader
 from rasa.core.domain import Domain
@@ -73,8 +75,7 @@ async def test_persist_and_read_test_story_graph(tmpdir, default_domain):
         "data/test_stories/stories.md", default_domain
     )
     out_path = tmpdir.join("persisted_story.md")
-    with open(out_path.strpath, "w", encoding="utf-8") as f:
-        f.write(graph.as_story_string())
+    rasa.utils.io.write_text_file(graph.as_story_string(), out_path.strpath)
 
     recovered_trackers = await training.load_data(
         out_path.strpath,
@@ -333,13 +334,9 @@ async def test_load_multi_file_training_data(default_domain):
 
 async def test_load_training_data_handles_hidden_files(tmpdir, default_domain):
     # create a hidden file
-
-    with open(os.path.join(tmpdir.strpath, ".hidden"), "a") as f:
-        f.close()
+    Path(tmpdir / ".hidden").touch()
     # create a normal file
-    normal_file = os.path.join(tmpdir.strpath, "normal_file")
-    with open(normal_file, "a") as f:
-        f.close()
+    Path(tmpdir / "normal_file").touch()
 
     featurizer = MaxHistoryTrackerFeaturizer(
         BinarySingleStateFeaturizer(), max_history=2

tests/core/test_visualization.py

@@ -1,5 +1,6 @@
 from rasa.core.events import ActionExecuted, SlotSet, UserUttered
 from rasa.core.training import visualization
+import rasa.utils.io
 
 
 def test_style_transfer():
@@ -94,8 +95,7 @@ async def test_graph_persistence(default_domain, tmpdir):
 
     assert isfile(out_file)
 
-    with open(out_file, "r") as graph_file:
-        content = graph_file.read()
+    content = rasa.utils.io.read_file(out_file)
 
     assert "isClient = true" in content
     assert "graph = `{}`".format(generated_graph.to_string()) in content

tests/test_server.py

@@ -19,6 +19,7 @@
 
 import rasa
 import rasa.constants
+import rasa.utils.io
 from rasa.core import events, utils
 from rasa.core.agent import Agent
 from rasa.core.channels import CollectingOutputChannel, RestInput, SlackInput
@@ -453,8 +454,7 @@ def test_train_internal_error(rasa_app: SanicTestClient):
 
 
 def test_evaluate_stories(rasa_app, default_stories_file):
-    with open(default_stories_file, "r") as f:
-        stories = f.read()
+    stories = rasa.utils.io.read_file(default_stories_file)
 
     _, response = rasa_app.post("/model/test/stories", data=stories)
 
@@ -482,17 +482,15 @@ def test_evaluate_stories(rasa_app, default_stories_file):
 def test_evaluate_stories_not_ready_agent(
     rasa_app_nlu: SanicTestClient, default_stories_file
 ):
-    with open(default_stories_file, "r") as f:
-        stories = f.read()
+    stories = rasa.utils.io.read_file(default_stories_file)
 
     _, response = rasa_app_nlu.post("/model/test/stories", data=stories)
 
     assert response.status == 409
 
 
 def test_evaluate_stories_end_to_end(rasa_app, end_to_end_story_file):
-    with open(end_to_end_story_file, "r") as f:
-        stories = f.read()
+    stories = rasa.utils.io.read_file(end_to_end_story_file)
 
     _, response = rasa_app.post("/model/test/stories?e2e=true", data=stories)
 
@@ -517,8 +515,7 @@ def test_evaluate_stories_end_to_end(rasa_app, end_to_end_story_file):
 
 
 def test_evaluate_intent(rasa_app, default_nlu_data):
-    with open(default_nlu_data, "r") as f:
-        nlu_data = f.read()
+    nlu_data = rasa.utils.io.read_file(default_nlu_data)
 
     _, response = rasa_app.post("/model/test/intents", data=nlu_data)
 
@@ -533,8 +530,7 @@ def test_evaluate_intent(rasa_app, default_nlu_data):
 def test_evaluate_intent_on_just_nlu_model(
     rasa_app_nlu: SanicTestClient, default_nlu_data
 ):
-    with open(default_nlu_data, "r") as f:
-        nlu_data = f.read()
+    nlu_data = rasa.utils.io.read_file(default_nlu_data)
 
     _, response = rasa_app_nlu.post("/model/test/intents", data=nlu_data)
 
@@ -552,8 +548,7 @@ def test_evaluate_intent_with_query_param(
     _, response = rasa_app.get("/status")
     previous_model_file = response.json["model_file"]
 
-    with open(default_nlu_data, "r") as f:
-        nlu_data = f.read()
+    nlu_data = rasa.utils.io.read_file(default_nlu_data)
 
     _, response = rasa_app.post(
         f"/model/test/intents?model={trained_nlu_model}", data=nlu_data

tests/utils/test_io.py

@@ -146,9 +146,7 @@ def test_emojis_in_tmp_file():
             - two £ (?u)\\b\\w+\\b f\u00fcr
         """
     test_file = io_utils.create_temporary_file(test_data)
-    with open(test_file, mode="r", encoding="utf-8") as f:
-        content = f.read()
-    content = io_utils.read_yaml(content)
+    content = io_utils.read_yaml_file(test_file)
 
     assert content["data"][0] == "one 😁💯 👩🏿‍💻👨🏿‍💻"
     assert content["data"][1] == "two £ (?u)\\b\\w+\\b für"