Merge remote-tracking branch 'origin/master' into auto-push-full-impl

.circleci/config.yml

@@ -145,6 +145,26 @@ jobs:
       - run: mix dialyzer
       - save_plt_cache
 
+  test_performance: 
+    machine: true
+    resource_class: membraneframework/core-performance-test
+    working_directory: ~/app
+
+    steps:
+      - attach_workspace:
+          at: .
+      - run: cp -r benchmark/ ~/benchmark_backup/
+      - run: cp mix.exs ~/benchmark_backup/
+      - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/run.exs /root/results/results.res
+      - run: git checkout master
+      - run: cp  ~/benchmark_backup/mix.exs ~/app
+      - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -v ~/benchmark_backup/benchmark:/root/app/benchmark -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/run.exs /root/results/results_ref.res
+      - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -v ~/benchmark_backup/benchmark:/root/app/benchmark -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/compare.exs /root/results/results.res /root/results/results_ref.res
+      - run: 
+          command: rm ~/results/*
+          when: always
+
+
 workflows:
   version: 2
   build_and_test:
@@ -159,3 +179,9 @@ workflows:
       - lint:
           requires:
             - get_deps
+      - request_performance_test:
+          type: approval
+      - test_performance:
+          requires:
+            - get_deps
+            - request_performance_test

.formatter.exs

@@ -10,7 +10,7 @@ locals_without_parens =
 
 [
   inputs: [
-    "{lib,test,config}/**/*.{ex,exs}",
+    "{lib,test,config,benchmark}/**/*.{ex,exs}",
     "mix.exs"
   ],
   locals_without_parens: locals_without_parens,

benchmark/compare.exs

@@ -0,0 +1,53 @@
+# A script providing a functionality to compare results of two performance tests.
+
+# Comparison of two test results is done with the following command:
+# `mix run benchmark/compare.exs <result file> <reference result file>`
+# where the "result files" are the files generated with `mix run benchmark/run.exs` script.
+# For information about the metric used, see the modules implementing `Benchmark.Metric` behaviour.
+
+defmodule Benchmark.Compare do
+  require Logger
+
+  def run(results, ref_results) do
+    if Map.keys(results) != Map.keys(ref_results),
+      do: raise("Incompatible performance test result files!")
+
+    Enum.each(Map.keys(results), fn test_case ->
+      test_case_results = Map.get(results, test_case)
+      test_case_results_ref = Map.get(ref_results, test_case)
+
+      results_str =
+        Enum.map(Map.keys(test_case_results), fn metric_module ->
+          """
+          METRIC: #{metric_module}
+          #{inspect(Map.get(test_case_results, metric_module), pretty: true)}
+          vs
+          #{inspect(Map.get(test_case_results_ref, metric_module), pretty: true)}
+
+          """
+        end)
+        |> Enum.join()
+
+      Logger.debug("""
+      TEST CASE:
+      #{inspect(test_case, pretty: true)}
+
+      #{results_str}
+
+      """)
+
+      Enum.each(Map.keys(test_case_results), fn metric_module ->
+        metric_value = Map.get(test_case_results, metric_module)
+        metric_value_ref = Map.get(test_case_results_ref, metric_module)
+        metric_module.assert(metric_value, metric_value_ref, test_case)
+      end)
+    end)
+
+    :ok
+  end
+end
+
+[results_filename, ref_results_filename] = System.argv() |> Enum.take(2)
+results = File.read!(results_filename) |> :erlang.binary_to_term()
+ref_results = File.read!(ref_results_filename) |> :erlang.binary_to_term()
+Benchmark.Compare.run(results, ref_results)

benchmark/metric.ex

@@ -0,0 +1,34 @@
+defmodule Benchmark.Metric do
+  @moduledoc """
+  A module defining a behaviour for metrics used in Membrane Core performance benchmarks.
+  """
+
+  @typedoc """
+  A type describing a single meassurement of a given metric.
+
+  """
+  @type meassurement :: any
+
+  @opaque meassurement_state :: any
+
+  @doc """
+  A function used to assert that the first meassurement is no worse than the second meassurement.
+  """
+  @callback assert(
+              first_meassurement :: meassurement(),
+              second_meassurement :: meassurement(),
+              additional_params :: any
+            ) ::
+              :ok | no_return
+
+  @doc """
+  A function aggregating the list of meassurements gathered during multiple runs of the benchamark into a single
+  meassurement.
+
+  In case the first meassurement is worse than the second, the function should raise.
+  """
+  @callback average([meassurement]) :: meassurement
+
+  @callback start_meassurement(opts :: any) :: meassurement_state
+  @callback stop_meassurement(meassurement_state) :: meassurement
+end

benchmark/metric/final_memory.ex

@@ -0,0 +1,32 @@
+defmodule Benchmark.Metric.FinalMemory do
+  @behaviour Benchmark.Metric
+
+  @allowed_worsening_factor 0.5
+
+  @impl true
+  def assert(final_memory, final_memory_ref, test_case) do
+    if final_memory > final_memory_ref * (1 + @allowed_worsening_factor),
+      do:
+        raise(
+          "The memory performance has got worse! For test case: #{inspect(test_case, pretty: true)}
+        the final memory used to be: #{final_memory_ref} MB and now it is: #{final_memory} MB"
+        )
+
+    :ok
+  end
+
+  @impl true
+  def average(final_memory_samples) do
+    Enum.sum(final_memory_samples) / (length(final_memory_samples) * 1_000_000)
+  end
+
+  @impl true
+  def start_meassurement(_opts \\ nil) do
+    :erlang.memory(:total)
+  end
+
+  @impl true
+  def stop_meassurement(starting_memory) do
+    :erlang.memory(:total) - starting_memory
+  end
+end

benchmark/metric/in_progress_memory.ex

@@ -0,0 +1,61 @@
+defmodule Benchmark.Metric.InProgressMemory do
+  @behaviour Benchmark.Metric
+
+  @allowed_worsening_factor 0.5
+  @sampling_period 100
+
+  @impl true
+  def assert(memory_samples, memory_samples_ref, test_case) do
+    cumulative_memory = integrate(memory_samples)
+    cumulative_memory_ref = integrate(memory_samples_ref)
+
+    if cumulative_memory > cumulative_memory_ref * (1 + @allowed_worsening_factor),
+      do:
+        raise(
+          "The memory performance has got worse! For test case: #{inspect(test_case, pretty: true)}
+        the cumulative memory used to be: #{cumulative_memory_ref} MB and now it is: #{cumulative_memory} MB"
+        )
+
+    :ok
+  end
+
+  defp integrate(memory_samples) do
+    Enum.sum(memory_samples)
+  end
+
+  @impl true
+  def average(memory_samples_from_multiple_tries) do
+    memory_samples_from_multiple_tries
+    |> Enum.zip()
+    |> Enum.map(&Tuple.to_list(&1))
+    |> Enum.map(&(Enum.sum(&1) / (length(&1) * 1_000_000)))
+  end
+
+  @impl true
+  def start_meassurement(_opts \\ nil) do
+    initial_memory = :erlang.memory(:total)
+
+    task =
+      Task.async(fn ->
+        do_loop([], initial_memory)
+      end)
+
+    task
+  end
+
+  defp do_loop(acc, initial_memory) do
+    acc = acc ++ [:erlang.memory(:total) - initial_memory]
+
+    receive do
+      :stop -> acc
+    after
+      @sampling_period -> do_loop(acc, initial_memory)
+    end
+  end
+
+  @impl true
+  def stop_meassurement(task) do
+    send(task.pid, :stop)
+    Task.await(task)
+  end
+end

benchmark/metric/message_queues_length.ex

@@ -0,0 +1,70 @@
+defmodule Benchmark.Metric.MessageQueuesLength do
+  @behaviour Benchmark.Metric
+
+  @allowed_worsening_factor 0.5
+  @sampling_period 100
+
+  @impl true
+  def assert(queues_lengths, queues_lengths_ref, test_case) do
+    cumulative_queues_length = integrate(queues_lengths)
+    cumulative_queues_length_ref = integrate(queues_lengths_ref)
+
+    if cumulative_queues_length >
+         cumulative_queues_length_ref * (1 + @allowed_worsening_factor),
+       do:
+         raise(
+           "The cumulative queues length has got worse! For test case: #{inspect(test_case, pretty: true)}
+          the cumulative queues length to be: #{cumulative_queues_length_ref} and now it is: #{cumulative_queues_length}"
+         )
+
+    :ok
+  end
+
+  @impl true
+  def average(queues_lengths_from_multiple_tries) do
+    queues_lengths_from_multiple_tries
+    |> Enum.zip()
+    |> Enum.map(&Tuple.to_list(&1))
+    |> Enum.map(&(Enum.sum(&1) / length(&1)))
+  end
+
+  defp integrate(queues_lengths) do
+    Enum.sum(queues_lengths)
+  end
+
+  @impl true
+  def start_meassurement(children_pids) do
+    task =
+      Task.async(fn ->
+        do_loop([], children_pids)
+      end)
+
+    task
+  end
+
+  defp do_loop(acc, children_pids) do
+    acc = acc ++ [meassure_queues_length(children_pids)]
+
+    receive do
+      :stop -> acc
+    after
+      @sampling_period -> do_loop(acc, children_pids)
+    end
+  end
+
+  defp meassure_queues_length(children_pids) do
+    Enum.map(children_pids, fn pid ->
+      case Process.info(self(), :message_queue_len) do
+        {:message_queue_len, message_queue_len} -> message_queue_len
+        _other -> 0
+      end
+    end)
+    |> Enum.sum()
+  end
+
+  @impl true
+  def stop_meassurement(task) do
+    send(task.pid, :stop)
+    Task.await(task)
+  end
+end

benchmark/metric/time.ex

@@ -0,0 +1,32 @@
+defmodule Benchmark.Metric.Time do
+  @behaviour Benchmark.Metric
+
+  @allowed_worsening_factor 0.1
+
+  @impl true
+  def assert(time, time_ref, test_case) do
+    if time > time_ref * (1 + @allowed_worsening_factor),
+      do:
+        raise(
+          "The time performance has got worse! For test case: #{inspect(test_case, pretty: true)} the test
+        used to take: #{time_ref} ms and now it takes: #{time} ms"
+        )
+
+    :ok
+  end
+
+  @impl true
+  def average(times) do
+    Enum.sum(times) / length(times)
+  end
+
+  @impl true
+  def start_meassurement(_opts \\ nil) do
+    :os.system_time(:milli_seconds)
+  end
+
+  @impl true
+  def stop_meassurement(starting_time) do
+    :os.system_time(:milli_seconds) - starting_time
+  end
+end