diff --git a/chesspp/engine.py b/chesspp/engine.py index 22bdc4c..5ce3b9a 100644 --- a/chesspp/engine.py +++ b/chesspp/engine.py @@ -25,6 +25,7 @@ class Limit: def __init__(self, time: float | None = None, nodes: int | None = None): self.time = time self.nodes = nodes + self.node_count = 0 def run(self, func, *args, **kwargs): """ @@ -36,6 +37,7 @@ class Limit: if self.nodes: self._run_nodes(func, *args, **kwargs) + self.node_count = self.nodes elif self.time: self._run_time(func, *args, **kwargs) @@ -47,6 +49,7 @@ class Limit: start = time.perf_counter_ns() while (time.perf_counter_ns() - start) / 1e9 < self.time: func(*args, **kwargs) + self.node_count += 1 def translate_to_engine_limit(self) -> chess.engine.Limit: if self.nodes: @@ -95,6 +98,7 @@ class BayesMctsEngine(Engine): def __init__(self, board: chess.Board, color: chess.Color, strategy: IStrategy): super().__init__(board, color, strategy) self.mcts = BayesianMcts(board, self.strategy, self.color) + self.node_counts = [] @staticmethod def get_name() -> str: @@ -103,7 +107,16 @@ class BayesMctsEngine(Engine): def play(self, board: chess.Board, limit: Limit) -> chess.engine.PlayResult: if len(board.move_stack) != 0: # apply previous move to mcts --> reuse previous simulation results self.mcts.apply_move(board.peek()) - limit.run(lambda: self.mcts.sample(1)) + + node_count = 0 + + def do(): + nonlocal node_count + self.mcts.sample(1) + node_count += 1 + + limit.run(do) + self.node_counts.append(node_count) best_move = self.get_best_move(self.mcts.get_moves(), board.turn) self.mcts.apply_move(best_move) return chess.engine.PlayResult(move=best_move, ponder=None) @@ -121,6 +134,7 @@ class BayesMctsEngine(Engine): class ClassicMctsEngine(Engine): def __init__(self, board: chess.Board, color: chess.Color, strategy: IStrategy): super().__init__(board, color, strategy) + self.node_counts = [] @staticmethod def get_name() -> str: @@ -128,7 +142,15 @@ class ClassicMctsEngine(Engine): def play(self, board: chess.Board, limit: Limit) -> chess.engine.PlayResult: mcts_root = ClassicMcts(board, self.color, self.strategy) - limit.run(lambda: mcts_root.build_tree(1)) + node_count = 0 + + def do(): + nonlocal node_count + mcts_root.build_tree(1) + node_count += 1 + + limit.run(do) + self.node_counts.append(node_count) best_move = max(mcts_root.children, key=lambda x: x.score).move if board.turn == chess.WHITE else ( min(mcts_root.children, key=lambda x: x.score).move) return chess.engine.PlayResult(move=best_move, ponder=None) diff --git a/chesspp/simulation.py b/chesspp/simulation.py index ff17211..3fc1b9d 100644 --- a/chesspp/simulation.py +++ b/chesspp/simulation.py @@ -1,5 +1,6 @@ import multiprocessing as mp import random +import time import chess import chess.pgn from typing import Tuple, List @@ -16,23 +17,61 @@ class Winner(Enum): Draw = 2 +@dataclass +class GameStatistics: + white: str + black: str + average_time_white: float + average_time_black: float + nodes_white: int + nodes_black: int + length: int + @dataclass class EvaluationResult: winner: Winner game: str + statistics: GameStatistics -def simulate_game(white: Engine, black: Engine, limit: Limit, board: chess.Board) -> chess.pgn.Game: +def simulate_game(white: Engine, black: Engine, limit: Limit, board: chess.Board) -> (chess.pgn.Game, GameStatistics): is_white_playing = True + times_white = [] + times_black = [] + game_length = 0 while not board.is_game_over(): + start = time.time() play_result = white.play(board, limit) if is_white_playing else black.play(board, limit) + end = time.time() + times_white.append(end - start) if is_white_playing else times_black.append(end - start) board.push(play_result.move) is_white_playing = not is_white_playing + game_length += 1 game = chess.pgn.Game.from_board(board) game.headers['White'] = white.get_name() game.headers['Black'] = black.get_name() - return game + + if hasattr(white, "node_counts"): + white_nodes = sum(white.node_counts) // len(white.node_counts) + else: + white_nodes = 0 + + if hasattr(black, "node_counts"): + black_nodes = sum(black.node_counts) // len(black.node_counts) + else: + black_nodes = 0 + + statistics = GameStatistics(white=white.get_name(), + black=black.get_name(), + average_time_white=(sum(times_white)/len(times_white)), + average_time_black=(sum(times_black)/len(times_black)), + nodes_white=white_nodes, + nodes_black=black_nodes, + length=game_length + ) + + return game, statistics class Evaluation: @@ -73,7 +112,7 @@ class Evaluation: stockfish_path, lc0_path, stockfish_elo), EngineFactory.create_engine( engine_b, strategy_b, chess.BLACK, stockfish_path, lc0_path, stockfish_elo) - game = simulate_game(white, black, limit, chess.Board()) + game, statistics = simulate_game(white, black, limit, chess.Board()) winner = game.end().board().outcome().winner result = Winner.Draw @@ -87,4 +126,4 @@ class Evaluation: case (chess.BLACK, False): result = Winner.Engine_B - return EvaluationResult(result, str(game)) + return EvaluationResult(result, str(game), statistics) diff --git a/main.py b/main.py index fc7399e..93d79c3 100644 --- a/main.py +++ b/main.py @@ -95,6 +95,21 @@ def test_evaluation(): evaluator = simulation.Evaluation(a, s1, b, s2, limit, stockfish_path, lc0_path, stockfish_elo) results = evaluator.run(n, proc) + + for r in results: + stats = r.statistics + print("====================================") + print(f"Game length: {stats.length} moves") + print(f"{stats.white} (White):") + print(f"Average node count: {stats.nodes_white}") + print(f"Average simulation time: {stats.average_time_white}") + print() + print(f"{stats.black} (Black):") + print(f"Average node count: {stats.nodes_black}") + print(f"Average simulation time: {stats.average_time_black}") + print("====================================") + print() + games_played = len(results) a_wins = len(list(filter(lambda x: x.winner == simulation.Winner.Engine_A, results))) b_wins = len(list(filter(lambda x: x.winner == simulation.Winner.Engine_B, results)))