From 2c1de3969f809cfcb71e701543fa590499d2fd05 Mon Sep 17 00:00:00 2001 From: Oceankoh <36119714+Oceankoh@users.noreply.github.com> Date: Sun, 21 Jan 2024 03:30:08 +0800 Subject: [PATCH] merge conflicts --- backend/src/chess/engine.ts | 12 ++ backend/src/chess/llm.ts | 236 ++++++++++++++++++------------------ backend/test/llm.js | 121 ++++++++++-------- 3 files changed, 204 insertions(+), 165 deletions(-) diff --git a/backend/src/chess/engine.ts b/backend/src/chess/engine.ts index 7affc65..bd3902e 100644 --- a/backend/src/chess/engine.ts +++ b/backend/src/chess/engine.ts @@ -9,6 +9,10 @@ class NormalMove { this.square1 = square1; this.square2 = square2; } + + toString(): string { + return `(${this.square1}, ${this.square2})`; + } } class PromotionMove extends NormalMove { @@ -17,6 +21,10 @@ class PromotionMove extends NormalMove { super(square1, square2); this.piece = piece; } + + toString(): string { + return `(${this.square1}, ${this.square2}, '${this.piece}')`; + } } class InvalidMove { @@ -24,6 +32,10 @@ class InvalidMove { constructor(prompt: string) { this.prompt = prompt; } + + toString() { + return 'Invalid Move' + } } class FailedMove { diff --git a/backend/src/chess/llm.ts b/backend/src/chess/llm.ts index b37ff63..bd57361 100644 --- a/backend/src/chess/llm.ts +++ b/backend/src/chess/llm.ts @@ -6,14 +6,14 @@ import { InvalidMove, Move, NormalMove, PromotionMove } from "./engine.js"; dotenv.config(); async function displayTokenCount(model, request) { - const { totalTokens } = await model.countTokens(request); - console.log("Token count: ", totalTokens); + const { totalTokens } = await model.countTokens(request); + console.log("Token count: ", totalTokens); } async function displayChatTokenCount(model, chat, msg) { - const history = await chat.getHistory(); - const msgContent = { role: "user", parts: [{ text: msg }] }; - await displayTokenCount(model, { contents: [...history, msgContent] }); + const history = await chat.getHistory(); + const msgContent = { role: "user", parts: [{ text: msg }] }; + await displayTokenCount(model, { contents: [...history, msgContent] }); } // Access your API key as an environment variable (see "Set up your API key" above) @@ -21,124 +21,130 @@ const genAI = new GoogleGenerativeAI(process.env.GEMINI_KEY); const FEN = "rnbqkb1r/1p2ppp1/3p4/p2n3p/3P4/3B1N2/PPP2PPP/RNBQK2R w KQkq - 0 7"; export async function llmInterpretPrompt( - prompt: string, - fen: string + prompt: string, + fen: string ): Promise { - // For text-only input, use the gemini-pro model - const model = genAI.getGenerativeModel({ model: "gemini-pro" }); - const chat = model.startChat({ - history: [ - { - role: "user", - parts: `Assume the role of an Next Generation Chess Interpreter. Players will describe their moves within 15 words and you are to parse them into valid chess moves. Your response can be one of the following: - - 1. (square, square), to move a piece from the first square to the second square. For example, ('e2', 'e4') - 2. (square, square, piece), to promote a pawn to a piece. For example, ('e7', 'e8', 'q'). This promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). - 3. 'Invalid move', if the move does not make sense or is illegal. - If you understand, respond with 'Yes, I understand'. The current game state is provided by the following FEN: ${fen}`, - }, - { - role: "model", - parts: "Yes, I understand.", - }, - ], - generationConfig: { - maxOutputTokens: 1000, - }, - }); - - const result = await chat.sendMessage(prompt); - const response = await result.response; - - if ( - response.promptFeedback && - response.promptFeedback.blockReason !== - BlockReason.BLOCKED_REASON_UNSPECIFIED - ) { - return new InvalidMove( - "Blocked Prompt: " + response.promptFeedback.blockReasonMessage - ); - } - - const text = response.text(); - const safe = await llmCheckMoveValidity(text, fen); - console.log(parseResponseMove(text), safe); - if (safe) { - return parseResponseMove(text); - } else { - return new InvalidMove("Illegal Move: " + text); - } + // For text-only input, use the gemini-pro model + const model = genAI.getGenerativeModel({ model: "gemini-pro" }); + const chat = model.startChat({ + history: [ + { + role: "user", + parts: `Assume the role of an Next Generation Chess Interpreter. Players will describe their moves and you are to parse them into valid chess moves. + + The current game state is provided by the following FEN: ${fen} + + Your response must be one of the following: + + 1. (, ), to move a piece from the first square to the second square. For example, ('e2', 'e4') + 2. (, , ), to promote a pawn to a piece. For example, ('e7', 'e8', 'q'). This promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). + + This is very important: You should only have either a move formatted as (, ) or (, , ) in your response. + + If you understand, respond with 'Yes, I understand'.`, + }, + { + role: "model", + parts: "Yes, I understand.", + }, + ], + generationConfig: { + maxOutputTokens: 500, + }, + }); + + const result = await chat.sendMessage(prompt); + const response = await result.response; + + if ( + response.promptFeedback && + response.promptFeedback.blockReason !== + BlockReason.BLOCKED_REASON_UNSPECIFIED + ) { + return new InvalidMove( + "Blocked Prompt: " + response.promptFeedback.blockReasonMessage + ); + } + + const text = response.text(); + const parsed = parseResponseMove(text); + if (parsed instanceof InvalidMove) { + return parsed; + } + const safe = await llmCheckMoveValidity(parsed, fen); + console.log(parseResponseMove(text), safe); + if (safe) { + return parseResponseMove(text); + } else { + return new InvalidMove(`Illegal Move: ${text}`); + } } async function llmCheckMoveValidity( - prompt: string, - fen: string + prompt: NormalMove | PromotionMove, + fen: string ): Promise { - // For text-only input, use the gemini-pro model - const model = genAI.getGenerativeModel({ model: "gemini-pro" }); - const chat = model.startChat({ - history: [ - { - role: "user", - parts: `Assume the role of an Next Generation Chess Interpreter. Given the FEN of the current game, you are to determine whether a move is legal. The input can be one of the following formats: - - 1. (square, square), to move a piece from the first square to the second square. For example, ('e2', 'e4') moves the piece at e2 to e4. - 2. (square, square, piece), to promote a pawn to a piece. For example, ('e7', 'e8', 'q'), promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). - - If the move is legal, respond with 'True'. If the move is illegal, respond with 'False'. You should only have either 'True' or 'False' in your response. - If you understand, respond with 'Yes, I understand'. The current game state is provided by the following FEN: ${fen}`, - }, - { - role: "model", - parts: "Yes, I understand.", - }, - ], - generationConfig: { - maxOutputTokens: 100, - }, - }); - - const result = await chat.sendMessage(prompt); - const response = await result.response; - - if ( - response.promptFeedback && - response.promptFeedback.blockReason !== - BlockReason.BLOCKED_REASON_UNSPECIFIED - ) { - return false; - } - - const text = response.text(); - return text === "True"; + // For text-only input, use the gemini-pro model + const model = genAI.getGenerativeModel({ model: "gemini-pro" }); + const chat = model.startChat({ + history: [ + { + role: "user", + parts: `Assume the role of an Next Generation Chess Interpreter. Using the FEN and next move you are to determine whether the next move is legal. The move can be one of the following formats: + + 1. (, ), to move a piece from the first square to the second square. For example, (e2, e4) moves the piece at e2 to e4. + 2. (, , ), to promote a pawn to a piece. For example, (e7, e8, q), promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). + + If the move is legal, respond with 'True'. If the move is illegal, respond with 'False'. You should only have either 'True' or 'False' in your response. + If you understand, respond with 'Yes, I understand'.`, + }, + { + role: "model", + parts: "Yes, I understand.", + }, + ], + generationConfig: { + maxOutputTokens: 100, + }, + }); + + const result = await chat.sendMessage( + `The current game state is provided by the following FEN: ${fen}. The move to be made is ${prompt.toString()}` + ); + + const response = await result.response; + if ( + response.promptFeedback && + response.promptFeedback.blockReason !== + BlockReason.BLOCKED_REASON_UNSPECIFIED + ) { + return false; + } + + const text = response.text(); + return text === "True"; } function parseResponseMove(response: string): Move { - // if response contains 'Invalid move', return 'Invalid move' - if (response.includes("Invalid move")) { - return new InvalidMove(response); - } - - // check if response is in the format (square, square) - const moveRegex = /\(\'([abcdefgh]\d)\', \'([abcdefgh]\d)\'\)/; - const moveMatch = response.match(moveRegex); - if (moveMatch) { - const [_, square1, square2] = moveMatch; - return new NormalMove(square1, square2); - } - - // check if response is in the format (square, square) - const promotionRegex = - /\(\'([abcdefgh]\d)\', \'([abcdefgh])\d\', '([qrbn])'\)/; - const promotionMatch = response.match(promotionRegex); - if (promotionMatch) { - const [_, square1, square2, piece] = promotionMatch; - if (piece === "q" || piece === "r" || piece === "b" || piece === "n") { - return new PromotionMove(square1, square2, piece); - } else { - assertNever(); + // check if response is in the format (square, square) + const moveRegex = /\(\'?([abcdefgh]\d)\'?,\s?\'?([abcdefgh]\d)\'?\)/; + const moveMatch = response.match(moveRegex); + if (moveMatch) { + const [_, square1, square2] = moveMatch; + return new NormalMove(square1, square2); } - } - return new InvalidMove(`Illegal Move: \n ${response}`); + // check if response is in the format (square, square) + const promotionRegex = + /\('?([abcdefgh]\d)'?,\s?'?([abcdefgh]\d)'?,\s?'?([qrbn])'?\)/; + const promotionMatch = response.match(promotionRegex); + if (promotionMatch) { + const [_, square1, square2, piece] = promotionMatch; + if (piece === "q" || piece === "r" || piece === "b" || piece === "n") { + return new PromotionMove(square1, square2, piece); + } else { + assertNever(); + } + } + return new InvalidMove(`Invalid Response: ${response}`); } diff --git a/backend/test/llm.js b/backend/test/llm.js index 86010c4..0badbff 100644 --- a/backend/test/llm.js +++ b/backend/test/llm.js @@ -1,5 +1,5 @@ import { GoogleGenerativeAI } from "@google/generative-ai"; -import dotenv from 'dotenv'; +import dotenv from "dotenv"; dotenv.config(); async function displayTokenCount(model, request) { @@ -18,41 +18,51 @@ const genAI = new GoogleGenerativeAI(process.env.GEMINI_KEY); const FEN = "rnbqkb1r/1p2ppp1/3p4/p2n3p/3P4/3B1N2/PPP2PPP/RNBQK2R w KQkq - 0 7"; async function run(prompt, fen) { + console.log("prompt: ", prompt); // For text-only input, use the gemini-pro model const model = genAI.getGenerativeModel({ model: "gemini-pro" }); const chat = model.startChat({ history: [ - { - role: "user", - parts: `Assume the role of an Next Generation Chess Interpreter. Players will describe their moves within 15 words and you are to parse them into valid chess moves. Your response can be one of the following: - - 1. (square, square), to move a piece from the first square to the second square. For example, ('e2', 'e4') - 2. (square, square, piece), to promote a pawn to a piece. For example, ('e7', 'e8', 'q'). This promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). - 3. 'Invalid move', if the move does not make sense or is illegal. - If you understand, respond with 'Yes, I understand'. The current game state is provided by the following FEN: ${fen}`, - }, - { - role: "model", - parts: "Yes, I understand.", - }, + { + role: "user", + parts: `Assume the role of an Next Generation Chess Interpreter. Players will describe their moves and you are to parse them into valid chess moves. + + The current game state is provided by the following FEN: ${fen} + + Your response must be one of the following: + + 1. (, ), to move a piece from the first square to the second square. For example, ('e2', 'e4') + 2. (, , ), to promote a pawn to a piece. For example, ('e7', 'e8', 'q'). This promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). + + This is very important: You should only have either a move formatted as (, ) or (, , ) in your response. + + If you understand, respond with 'Yes, I understand'.`, + }, + { + role: "model", + parts: "Yes, I understand.", + }, ], generationConfig: { - maxOutputTokens: 1000, + maxOutputTokens: 1000, }, }); try { - const result = await chat.sendMessage(prompt) + const result = await chat.sendMessage(prompt); const response = await result.response; const text = response.text(); - const safe = await check(text); - console.log(parseResponse(text), safe); + console.log("before parsing: ", text); + const parsed = parseResponse(text); + console.log("after parsing: ", parsed); + const safe = await check(parsed, fen); if (safe) { - return parseResponse(text); + return parsed; } } catch (e) { console.log(e); } + console.log(`\n\n\n\n`); } async function check(prompt, fen) { @@ -60,31 +70,35 @@ async function check(prompt, fen) { const model = genAI.getGenerativeModel({ model: "gemini-pro" }); const chat = model.startChat({ history: [ - { - role: "user", - parts: `Assume the role of an Next Generation Chess Interpreter. Given the FEN of the current game, you are to determine whether a move is legal. The input can be one of the following formats: - - 1. (square, square), to move a piece from the first square to the second square. For example, ('e2', 'e4') moves the piece at e2 to e4. - 2. (square, square, piece), to promote a pawn to a piece. For example, ('e7', 'e8', 'q'), promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). - - If the move is legal, respond with 'True'. If the move is illegal, respond with 'False'. You should only have either 'True' or 'False' in your response. - If you understand, respond with 'Yes, I understand'. The current game state is provided by the following FEN: ${fen}`, - }, - { - role: "model", - parts: "Yes, I understand.", - }, + { + role: "user", + parts: `Assume the role of an Next Generation Chess Interpreter. Using the FEN and next move you are to determine whether the next move is legal. The move can be one of the following formats: + + 1. (, ), to move a piece from the first square to the second square. For example, (e2, e4) moves the piece at e2 to e4. + 2. (, , ), to promote a pawn to a piece. For example, (e7, e8, q), promotes the pawn at e7 to a queen. The piece can be a 'q' (queen), 'r' (rook), 'b' (bishop), or 'n' (knight). + + If the move is legal, respond with 'True'. If the move is illegal, respond with 'False'. You should only have either 'True' or 'False' in your response. + If you understand, respond with 'Yes, I understand'.`, + }, + { + role: "model", + parts: "Yes, I understand.", + }, ], generationConfig: { - maxOutputTokens: 100, + maxOutputTokens: 100, }, }); try { - const result = await chat.sendMessage(prompt) + console.log(fen); + const result = await chat.sendMessage( + `The current game state is provided by the following FEN: ${fen}. The move to be made is ${prompt}` + ); const response = await result.response; const text = response.text(); - return text === 'True'; + console.log("safety check", text); + return text === "True"; } catch (e) { console.log(e); } @@ -92,39 +106,46 @@ async function check(prompt, fen) { function parseResponse(response) { console.log(response); + response = response.trim(); // accept only (square, square) or (square, square, piece) or 'Invalid move' - + // if response contains 'Invalid move', return 'Invalid move' - if (response.includes('Invalid move')) { - return response; + if (response.includes("Invalid move")) { + // return response; + return "Invalid Move"; } // check if response is in the format (square, square) - const moveRegex = /\(\'([abcdefgh]\d)\', \'([abcdefgh]\d)\'\)/; + const moveRegex = /\(\'?([abcdefgh]\d)\'?,\s?\'?([abcdefgh]\d)\'?\)/; const moveMatch = response.match(moveRegex); if (moveMatch) { const [_, square1, square2] = moveMatch; - return {square1, square2}; + // return { square1, square2 }; + return `(${square1}, ${square2})`; } - + // check if response is in the format (square, square) - const promotionRegex = /\(\'([abcdefgh]\d)\', \'([abcdefgh])\d\', '([qrbn])'\)/; + const promotionRegex = + /\('?([abcdefgh]\d)'?,\s?'?([abcdefgh]\d)'?,\s?'?([qrbn])'?\)/; const promotionMatch = response.match(promotionRegex); if (promotionMatch) { const [_, square1, square2, piece] = promotionMatch; - return {square1, square2, piece}; + // return { square1, square2, piece }; + return `(${square1}, ${square2}, ${piece})`; } - return `Illegal Response: \n ${response}`; - + // return `Illegal Response: \n ${response}`; + return `Illegal Response`; } // user prompt const prompt1 = "capture the opponent's rook"; const prompt2 = "advance and promote all my pawns"; const prompt3 = "deliver a checkmate"; const prompt4 = "('e2', 'e8', 'q')"; +const prompt5 = "move the piece at d1 to d2"; -run(prompt1, FEN); -run(prompt2, FEN); -run(prompt3, FEN); -run(prompt4, FEN); +await run(prompt1, FEN); +await run(prompt2, FEN); +await run(prompt3, FEN); +await run(prompt4, FEN); +await run(prompt5, FEN);