add evaluation class for engines

engine.py

@@ -2,10 +2,10 @@ from abc import ABC, abstractmethod
 import chess
 import chess.engine
 from classic_mcts import ClassicMcts
+import random
 
 
 class Engine(ABC):
-
     color: chess.Color
     """The side the engine plays (``chess.WHITE`` or ``chess.BLACK``)."""
 
@@ -21,8 +21,9 @@ class Engine(ABC):
         """
         pass
 
+    @staticmethod
     @abstractmethod
-    def get_name(self) -> str:
+    def get_name() -> str:
         """
         Return the engine's name
         :return: the engine's name
@@ -34,7 +35,8 @@ class ClassicMctsEngine(Engine):
     def __init__(self, color: chess.Color):
         super().__init__(color)
 
-    def get_name(self) -> str:
+    @staticmethod
+    def get_name() -> str:
         return "ClassicMctsEngine"
 
     def play(self, board: chess.Board) -> chess.engine.PlayResult:
@@ -43,3 +45,16 @@ class ClassicMctsEngine(Engine):
         best_move = max(mcts_root.children, key=lambda x: x.score).move if board.turn == chess.WHITE else (
             min(mcts_root.children, key=lambda x: x.score).move)
         return chess.engine.PlayResult(move=best_move, ponder=None)
+
+
+class RandomEngine(Engine):
+    def __init__(self, color: chess.Color):
+        super().__init__(color)
+
+    @staticmethod
+    def get_name() -> str:
+        return "Random"
+
+    def play(self, board: chess.Board) -> chess.engine.PlayResult:
+        move = random.choice(list(board.legal_moves))
+        return chess.engine.PlayResult(move=move, ponder=None)

main.py

@@ -5,29 +5,13 @@ from classic_mcts import ClassicMcts
 import engine
 import eval
 import util
-
-
-def simulate_game(white: engine.Engine, black: engine.Engine) -> chess.pgn.Game:
-    board = chess.Board()
-
-    is_white_playing = True
-    while not board.is_game_over():
-        play_result = white.play(board) if is_white_playing else black.play(board)
-        board.push(play_result.move)
-        print(board)
-        print()
-        is_white_playing = not is_white_playing
-
-    game = chess.pgn.Game.from_board(board)
-    game.headers['White'] = white.get_name()
-    game.headers['Black'] = black.get_name()
-    return game
+import simulation
 
 
 def test_simulate():
     white = engine.ClassicMctsEngine(chess.WHITE)
     black = engine.ClassicMctsEngine(chess.BLACK)
-    game = simulate_game(white, black)
+    game = simulation.simulate_game(white, black)
     print(game)
 
 
@@ -77,8 +61,23 @@ def analyze_results(moves: dict):
         print(f"score for move {m}: manual_score={manual_score}, engine_score={engine_score}")
 
 
+def test_evaluation():
+    a = engine.ClassicMctsEngine
+    b = engine.RandomEngine
+    evaluator = simulation.Evaluation(a,b)
+    results = evaluator.run(4)
+    a_results = len(list(filter(lambda x: x.winner == simulation.Winner.Engine_A, results)))/len(results)*100
+    b_results = len(list(filter(lambda x: x.winner == simulation.Winner.Engine_B, results)))/len(results)*100
+    draws = len(list(filter(lambda x: x.winner == simulation.Winner.Draw, results)))/len(results)*100
+
+    print(f"Engine {a.get_name()} won {a_results}% of games")
+    print(f"Engine {b.get_name()} won {b_results}% of games")
+    print(f"{draws}% of games resulted in a draw")
+
+
 def main():
-    test_simulate()
+    test_evaluation()
+    # test_simulate()
     # test_mcts()
     # test_stockfish()
     # test_stockfish_prob()

simulation.py

@@ -0,0 +1,72 @@
+import multiprocessing as mp
+import random
+import chess
+import chess.pgn
+from typing import Tuple, List
+from enum import Enum
+from dataclasses import dataclass
+
+from engine import Engine
+
+
+class Winner(Enum):
+    Engine_A = 0
+    Engine_B = 1
+    Draw = 2
+
+
+@dataclass
+class EvaluationResult:
+    winner: Winner
+    game: chess.pgn.Game
+
+
+def simulate_game(white: Engine, black: Engine) -> chess.pgn.Game:
+    board = chess.Board()
+
+    is_white_playing = True
+    while not board.is_game_over():
+        play_result = white.play(board) if is_white_playing else black.play(board)
+        board.push(play_result.move)
+        is_white_playing = not is_white_playing
+
+    game = chess.pgn.Game.from_board(board)
+    game.headers['White'] = white.get_name()
+    game.headers['Black'] = black.get_name()
+    return game
+
+
+class Evaluation:
+    def __init__(self, engine_a: Engine.__class__, engine_b: Engine.__class__):
+        self.engine_a = engine_a
+        self.engine_b = engine_b
+
+    def run(self, n_games=100) -> List[EvaluationResult]:
+        with mp.Pool(mp.cpu_count()) as pool:
+            args = [(self.engine_a, self.engine_b) for i in range(n_games)]
+            return pool.map(Evaluation._test_simulate, args)
+
+    @staticmethod
+    def _test_simulate(arg: Tuple[Engine.__class__, Engine.__class__]) -> EvaluationResult:
+        engine_a, engine_b = arg
+        flip_engines = bool(random.getrandbits(1))
+        if flip_engines:
+            black, white = engine_a(chess.BLACK), engine_b(chess.WHITE)
+        else:
+            white, black = engine_a(chess.WHITE), engine_b(chess.BLACK)
+
+        game = simulate_game(white, black)
+        winner = game.end().board().outcome().winner
+
+        result = Winner.Draw
+        match (winner, flip_engines):
+            case (chess.WHITE, True):
+                result = Winner.Engine_B
+            case (chess.BLACK, True):
+                result = Winner.Engine_A
+            case (chess.WHITE, False):
+                result = Winner.Engine_A
+            case (chess.BLACK, False):
+                result = Winner.Engine_B
+
+        return EvaluationResult(result, game)