cleanup

.github/workflows/python-app.yml

@@ -31,4 +31,8 @@ jobs:
     - name: Test with pytest
       if: contains(github.event.head_commit.message, 'EVAL')
       run: |
-        python -m pytest ci_quixo/__main__.py -m evaluate -s
+        python -m ci_quixo
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v5
+      title: Generate nuove foto di eval
+      commit-message: updated results picture

ci_quixo/__main__.py

@@ -1,82 +1,53 @@
 try:
     from _players import MinMaxPlayer, MCTSPlayer, RandomPlayer
-    from helper import evaluate
+    from helper import evaluate, Result, gen_plots
+    from main import Player
 except:
     from ._players import *
-    from .helper import evaluate
-import pytest
+    from .main import Player
+    from .helper import evaluate, Result, gen_plots
 from pprint import pprint
+import dill, multiprocessing
+dill.Pickler.dumps, dill.Pickler.loads = dill.dumps, dill.loads
+multiprocessing.reduction.ForkingPickler = dill.Pickler
+multiprocessing.reduction.dump = dill.dump
 
 GAMES = 10
 HIDE_PBAR = True
 
-@pytest.mark.evaluate
-def test_minmax_vs_random():
-    minmax = MinMaxPlayer()
+minmax2 = MinMaxPlayer()
+minmax3 = MinMaxPlayer(3, pruning=2)
+mcts_r = MCTSPlayer()
+mcts_h = MCTSPlayer(sim_heuristic=True)
 
-    print("\n--- --- ---")
-    print(f"{minmax.name} VS RandomPlayer")
-    evaluate(minmax, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
-    print(f"{minmax.name} Stats:")
-    pprint(minmax.stats, sort_dicts=False)
 
-@pytest.mark.evaluate
-def test_minmax3_vs_random():
-    minmax = MinMaxPlayer(3, pruning=2)
+def test_minmax_vs_random() -> Result:
+    return evaluate(minmax2, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
 
-    print("\n--- --- ---")
-    print(f"{minmax.name} VS RandomPlayer")
-    evaluate(minmax, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
-    print(f"{minmax.name} Stats:")
-    pprint(minmax.stats, sort_dicts=False)
+def test_minmax3_vs_random() -> Result:
+    return evaluate(minmax3, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
 
-@pytest.mark.evaluate
-def test_mcts_r_vs_random():
-    mcts = MCTSPlayer()
+def test_mcts_r_vs_random() -> Result:
+    return evaluate(mcts_r, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
 
-    print("\n--- --- ---")
-    print(f"{mcts.name} vs RandomPlayer")
-    evaluate(mcts, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
-    print(f"{mcts.name} Stats:")
-    pprint(mcts.stats, sort_dicts=False)
 
+def test_mcts_h_vs_random() -> Result:
+    return evaluate(mcts_h, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
 
-@pytest.mark.evaluate
-def test_mcts_h_vs_random():
-    mcts = MCTSPlayer(sim_heuristic=True)
 
-    print("\n--- --- ---")
-    print(f"{mcts.name} vs RandomPlayer")
-    evaluate(mcts, RandomPlayer(), games=GAMES, display=True, hide_pbar=HIDE_PBAR)
-    print(f"{mcts.name} Stats:")
-    pprint(mcts.stats, sort_dicts=False)
+def test_minmax_vs_mcts_random() -> Result:
+    return evaluate(minmax2, mcts_r, games=GAMES, display=True, hide_pbar=HIDE_PBAR)
 
+def test_minmax_vs_mcts_heuristic() -> Result:
+    return evaluate(minmax2, mcts_h, games=GAMES, display=True, hide_pbar=HIDE_PBAR)
 
-@pytest.mark.evaluate
-def test_minmax_vs_mcts_random():
-    minmax = MinMaxPlayer()
-    mcts = MCTSPlayer()
-    print("\n--- --- ---")
-    print(f"{minmax.name} VS {mcts.name}")
-    evaluate(minmax, mcts, games=GAMES, display=True, hide_pbar=HIDE_PBAR)
-    print(f"{minmax.name} Stats:")
-    pprint(minmax.stats, sort_dicts=False)
-    print(f"{mcts.name} Stats:")
-    pprint(mcts.stats, sort_dicts=False)
-
-@pytest.mark.evaluate
-def test_minmax_vs_mcts_heuristic():
-    minmax = MinMaxPlayer()
-    mcts = MCTSPlayer(sim_heuristic=True)
-
-    print("\n--- --- ---")
-    print(f"{minmax.name} VS {mcts.name}")
-    evaluate(minmax, mcts, games=GAMES, display=True, hide_pbar=HIDE_PBAR)
-    print(f"{minmax.name} Stats:")
-    pprint(minmax.stats, sort_dicts=False)
-    print(f"{mcts.name} Stats:")
-    pprint(mcts.stats, sort_dicts=False)
-
+def call(it: callable) -> Result:
+    return it()
 
 if __name__ == "__main__":
-    print("Use `python -m pytest ci_quixo/__main__.py -m evaluate -s`")
+    evals = [test_minmax_vs_random, test_minmax3_vs_random, test_mcts_r_vs_random, test_mcts_h_vs_random, test_minmax_vs_mcts_random, test_minmax_vs_mcts_heuristic]
+
+    with multiprocessing.Pool() as p:
+        RESULTS = p.map(call, evals[:-2])
+
+    gen_plots(RESULTS)

ci_quixo/game.py

@@ -22,6 +22,11 @@ def __init__(self) -> None:
         '''You can change this for your player if you need to handle state/have memory'''
         pass
 
+    @property
+    def name(self) -> None:
+        '''Player Name'''
+        "Player"
+
     @abstractmethod
     def make_move(self, game: 'Game') -> tuple[tuple[int, int], Move]:
         '''

ci_quixo/helper.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TypedDict
 try:
     from main import Move, RandomPlayer, Game, Player
     from custom_game import CustomGame
@@ -10,8 +10,14 @@
     if TYPE_CHECKING:
         from .custom_game import CompleteMove
 
+import time
 import numpy as np
 from tqdm.auto import trange
+from os import path
+import matplotlib.pyplot as plt
+from dataclasses import dataclass, field
+from pprint import pformat
+PLOT_FOLDER = path.join(path.dirname(path.abspath(__file__)), "results")
 
 def pprint_board(game: "Game"):
     board: np.ndarray = game.get_board()
@@ -24,29 +30,120 @@ def pprint_board(game: "Game"):
             print(c, end="")
         print()
 
-def evaluate(p1: "Player", p2: "Player" = None, games: int = 10, display: bool = False, hide_pbar: bool = False) -> tuple[int]:
+@dataclass
+class Result:
+    p1: Player
+    p2: Player
+    wrfs: tuple[float]
+    """Player 1 Win Rate (as first, as second)"""
+    avg_time: float
+
+    @property
+    def wr(self) -> float:
+        return sum(self.wrfs)/2
+
+
+def evaluate(p1: "Player", p2: "Player" = None, games: int = 10, display: bool = False, hide_pbar: bool = False) -> Result:
     if games % 2 != 0:
         games += 1
     if p2 is None:
         p2 = RandomPlayer()
-    won_as_first, won_as_second = 0, 0
+    wins_as_first, wins_as_second = 0, 0
     pbar = trange(games, desc="Evaluating player", unit="game") if not hide_pbar else range(games)
+    p1name = "RandomPlayer" if isinstance(p1, RandomPlayer) else p1.name
+    p2name = "RandomPlayer" if isinstance(p2, RandomPlayer) else p2.name
+
+    durations = []
     for i in pbar:
         game = Game()
+        s = time.time()
         if i % 2 == 0:
-            won_as_first  += 1 if game.play(p1, p2) == 0 else 0
+            wins_as_first  += 1 if game.play(p1, p2) == 0 else 0
         else:
-            won_as_second += 1 if game.play(p2, p1) == 1 else 0
-        wins = won_as_first + won_as_second
+            wins_as_second += 1 if game.play(p2, p1) == 1 else 0
+        wins = wins_as_first + wins_as_second
+        durations.append(time.time() - s)
         not hide_pbar and pbar.set_postfix({"wins": f"{(wins/(i+1)):.1%}"})
     wins /= games
-    won_as_first /= games/2        
-    won_as_second /= games/2
+    wins_as_first /= games/2        
+    wins_as_second /= games/2
+    avg=sum(durations)/len(durations)
     if display:
-        print(f"Total wins : {wins:.2%}")
-        print(f"Wins as 1st: {won_as_first:.2%}")
-        print(f"Wins as 2nd: {won_as_second:.2%}")
-    return wins, won_as_first, won_as_second
+        print(f"----- {p1name} vs {p2name} w/ {games} games -----")
+        print(f"Total wins  : {wins:>6.2%}")
+        print(f"Wins as 1st : {wins_as_first:>6.2%}")
+        print(f"Wins as 2nd : {wins_as_second:>6.2%}")
+        print(f"Average Time: {avg:>6.2f}s")
+        try:
+            print(f"{p1.name} stats: \n{pformat(p1.stats)}")
+        except:
+            pass
+        try:
+            print(f"{p2.name} stats: \n{pformat(p2.stats)}")
+        except:
+            pass
+        print("----- ------")
+    # gen_plot("a", ("a", "a"), [1, 0])
+    return Result(p1, p2, (wins_as_first, wins_as_second), avg)
+
+
+def gen_plots(results: list[Result]) -> None:
+    plot_time_comparison(results)
+    plot_wr_vs_random(results)
+
+    pass
+
+def plot_wr_vs_random(results: list[Result]) -> None:
+    results = [r for r in results if isinstance(r.p2, RandomPlayer)]
+
+    player_names = [r.p1.short_name for r in results]
+    wr1 = [r.wrfs[0] * 100 for r in results]
+    wr2 = [r.wrfs[1] * 100 for r in results]
+    wr = [r.wr * 100 for r in results]
+
+    values = {
+        "Played as 1st": wr1,
+        "Played as 2nd": wr2,
+        "Overall": wr
+    }
+
+    x = np.arange(len(player_names)) * 1.6
+    width = 0.30  # the width of the bars
+
+
+    fig, ax = plt.subplots(layout='constrained')
+    multiplier = 0
+    for attribute, measurement in values.items():
+        offset = width * multiplier
+        rects = ax.bar(x + offset, measurement, width, label=attribute)
+        ax.bar_label(rects, padding=3)
+        multiplier += 1
+
+    # Add some text for labels, title and custom x-axis tick labels, etc.
+    ax.set_ylabel('Win Rate (%)')
+    ax.set_title('Agents Win Rate vs Random')
+    ax.set_xticks(x + width, player_names)
+    ax.legend(loc='upper left', ncols=3)
+    ax.set_ylim(0, 120)
+
+    plt.savefig(path.join(PLOT_FOLDER, "players_wr.png"))
+
+
+def plot_time_comparison(results: list[Result]) -> None:
+    # Only vs Random
+    results = sorted([r for r in results if isinstance(r.p2, RandomPlayer)], key=lambda it: it.avg_time)
+    ys = [r.avg_time for r in results]
+    xs = [r.p1.short_name for r in results]
+    plt_title = "Average Seconds per Game (vs Random)"
+    plt.plot(xs, ys, 'o')
+    plt.title(plt_title)
+    plt.ylabel("seconds")
+    plt.yticks(ys)
+    plt.xticks(list(range(len(xs))), xs, rotation=0)
+    plt.savefig(path.join(PLOT_FOLDER, "time_comparison.png"))
+
+
+
 
 class HumanPlayer(Player):
     def make_move(self, game: "Game") -> "CompleteMove":

ci_quixo/mcts.py

@@ -71,6 +71,9 @@ class MCTSPlayer(Player):
     _stats: dict[str, int] = field(default_factory=lambda: defaultdict(int), init=False)
     """Simple dict used to keep track of basic statistics, see property stats for a prettified version"""
 
+    @property
+    def short_name(self) -> str:
+        return f"MCTS({'H' if self.sim_heuristic else 'R'}, {self.games})"
 
     @property
     def name(self) -> str:

ci_quixo/minmax.py

@@ -64,6 +64,10 @@ def __init__(
 
         self._stats = defaultdict(int)
 
+    @property
+    def short_name(self) -> str:
+        return f"MinMax({'AB, ' if self.use_alpha_beta_pruning else ''}D{self.max_depth}, P{self.pruning_level}{', H' if self.use_htable else ''})"
+
     @property
     def name(self) -> str:
         return f"MinMax(depth={self.max_depth}, alpha_beta={self.use_alpha_beta_pruning}, pruning={self.pruning_level}, use_htable={self.use_htable})"
@@ -77,7 +81,7 @@ def make_move(self, game: Game) -> "CompleteMove":
             best_move = random.choice(cg.valid_moves())
         else:
             self._stats['evals'] += 1
-            self._stats['evals-ms'] += time.time() - start
+            self._stats['evals-ms'] += (time.time() - start)
 
 
         return best_move
@@ -236,7 +240,7 @@ def max_side(
     def _avg_time(self):
         if self._stats['evals'] == 0:
             return 0
-        return self._stats['eval-ms'] / self._stats['evals']
+        return self._stats['evals-ms'] / self._stats['evals']
 
 
     @property