{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import gym\n", "import gym_wordle\n", "from stable_baselines3 import DQN, PPO, common\n", "import numpy as np\n", "import tqdm" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ ">\n" ] } ], "source": [ "env = gym_wordle.wordle.WordleEnv()\n", "env = common.monitor.Monitor(env)\n", "\n", "print(env)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using cpu device\n", "Wrapping the env in a DummyVecEnv.\n", "---------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 5.59 |\n", "| time/ | |\n", "| fps | 544 |\n", "| iterations | 1 |\n", "| time_elapsed | 3 |\n", "| total_timesteps | 2048 |\n", "---------------------------------\n", "-----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 1.77 |\n", "| time/ | |\n", "| fps | 245 |\n", "| iterations | 2 |\n", "| time_elapsed | 16 |\n", "| total_timesteps | 4096 |\n", "| train/ | |\n", "| approx_kl | 0.021515464 |\n", "| clip_fraction | 0.335 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.47 |\n", "| explained_variance | 0.00118 |\n", "| learning_rate | 0.0003 |\n", "| loss | 89.5 |\n", "| n_updates | 10 |\n", "| policy_gradient_loss | -0.0854 |\n", "| value_loss | 262 |\n", "-----------------------------------------\n", "----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 1.31 |\n", "| time/ | |\n", "| fps | 211 |\n", "| iterations | 3 |\n", "| time_elapsed | 29 |\n", "| total_timesteps | 6144 |\n", "| train/ | |\n", "| approx_kl | 0.02457875 |\n", "| clip_fraction | 0.465 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.47 |\n", "| explained_variance | 0.161 |\n", "| learning_rate | 0.0003 |\n", "| loss | 118 |\n", "| n_updates | 20 |\n", "| policy_gradient_loss | -0.0987 |\n", "| value_loss | 217 |\n", "----------------------------------------\n", "----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 5.96 |\n", "| ep_rew_mean | 5.79 |\n", "| time/ | |\n", "| fps | 196 |\n", "| iterations | 4 |\n", "| time_elapsed | 41 |\n", "| total_timesteps | 8192 |\n", "| train/ | |\n", "| approx_kl | 0.02515613 |\n", "| clip_fraction | 0.447 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.47 |\n", "| explained_variance | 0.151 |\n", "| learning_rate | 0.0003 |\n", "| loss | 138 |\n", "| n_updates | 30 |\n", "| policy_gradient_loss | -0.103 |\n", "| value_loss | 242 |\n", "----------------------------------------\n", "-----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 4.9 |\n", "| time/ | |\n", "| fps | 177 |\n", "| iterations | 5 |\n", "| time_elapsed | 57 |\n", "| total_timesteps | 10240 |\n", "| train/ | |\n", "| approx_kl | 0.026685718 |\n", "| clip_fraction | 0.444 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.46 |\n", "| explained_variance | 0.176 |\n", "| learning_rate | 0.0003 |\n", "| loss | 96.7 |\n", "| n_updates | 40 |\n", "| policy_gradient_loss | -0.111 |\n", "| value_loss | 211 |\n", "-----------------------------------------\n", "----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 1.19 |\n", "| time/ | |\n", "| fps | 164 |\n", "| iterations | 6 |\n", "| time_elapsed | 74 |\n", "| total_timesteps | 12288 |\n", "| train/ | |\n", "| approx_kl | 0.02762504 |\n", "| clip_fraction | 0.463 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.46 |\n", "| explained_variance | 0.186 |\n", "| learning_rate | 0.0003 |\n", "| loss | 103 |\n", "| n_updates | 50 |\n", "| policy_gradient_loss | -0.115 |\n", "| value_loss | 200 |\n", "----------------------------------------\n", "----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 5.5 |\n", "| time/ | |\n", "| fps | 155 |\n", "| iterations | 7 |\n", "| time_elapsed | 92 |\n", "| total_timesteps | 14336 |\n", "| train/ | |\n", "| approx_kl | 0.02694263 |\n", "| clip_fraction | 0.458 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.46 |\n", "| explained_variance | 0.15 |\n", "| learning_rate | 0.0003 |\n", "| loss | 84.1 |\n", "| n_updates | 60 |\n", "| policy_gradient_loss | -0.116 |\n", "| value_loss | 225 |\n", "----------------------------------------\n", "-----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 7.27 |\n", "| time/ | |\n", "| fps | 154 |\n", "| iterations | 8 |\n", "| time_elapsed | 106 |\n", "| total_timesteps | 16384 |\n", "| train/ | |\n", "| approx_kl | 0.024316464 |\n", "| clip_fraction | 0.412 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.45 |\n", "| explained_variance | 0.173 |\n", "| learning_rate | 0.0003 |\n", "| loss | 126 |\n", "| n_updates | 70 |\n", "| policy_gradient_loss | -0.112 |\n", "| value_loss | 227 |\n", "-----------------------------------------\n", "-----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 7.8 |\n", "| time/ | |\n", "| fps | 151 |\n", "| iterations | 9 |\n", "| time_elapsed | 121 |\n", "| total_timesteps | 18432 |\n", "| train/ | |\n", "| approx_kl | 0.022988513 |\n", "| clip_fraction | 0.391 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.45 |\n", "| explained_variance | 0.206 |\n", "| learning_rate | 0.0003 |\n", "| loss | 139 |\n", "| n_updates | 80 |\n", "| policy_gradient_loss | -0.111 |\n", "| value_loss | 228 |\n", "-----------------------------------------\n", "-----------------------------------------\n", "| rollout/ | |\n", "| ep_len_mean | 6 |\n", "| ep_rew_mean | 6.14 |\n", "| time/ | |\n", "| fps | 153 |\n", "| iterations | 10 |\n", "| time_elapsed | 133 |\n", "| total_timesteps | 20480 |\n", "| train/ | |\n", "| approx_kl | 0.022813996 |\n", "| clip_fraction | 0.372 |\n", "| clip_range | 0.2 |\n", "| entropy_loss | -9.45 |\n", "| explained_variance | 0.199 |\n", "| learning_rate | 0.0003 |\n", "| loss | 117 |\n", "| n_updates | 90 |\n", "| policy_gradient_loss | -0.108 |\n", "| value_loss | 212 |\n", "-----------------------------------------\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "total_timesteps = 20_000\n", "model = PPO(\"MlpPolicy\", env, verbose=1, device='cuda')\n", "model.learn(total_timesteps=total_timesteps)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "model.save(\"dqn_wordle\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "c:\\Repository\\cse151b-final-project\\env\\Lib\\site-packages\\stable_baselines3\\common\\save_util.py:166: UserWarning: Could not deserialize object clip_range. Consider using `custom_objects` argument to replace this object.\n", "Exception: code() argument 13 must be str, not int\n", " warnings.warn(\n", "c:\\Repository\\cse151b-final-project\\env\\Lib\\site-packages\\stable_baselines3\\common\\save_util.py:166: UserWarning: Could not deserialize object lr_schedule. Consider using `custom_objects` argument to replace this object.\n", "Exception: code() argument 13 must be str, not int\n", " warnings.warn(\n" ] } ], "source": [ "model = PPO.load(\"dqn_wordle\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[16 1 9 19 5 3 2 3 3 1]\n", " [18 8 5 13 5 2 3 2 3 1]\n", " [16 1 9 19 5 3 2 3 3 1]\n", " [16 1 9 19 5 3 2 3 3 1]\n", " [16 1 9 19 5 3 2 3 3 1]\n", " [16 1 9 19 5 3 2 3 3 1]]\n", "[[16 1 9 19 5 3 3 3 3 3]\n", " [18 8 5 13 5 3 2 3 3 3]\n", " [16 1 9 19 5 3 3 3 3 3]\n", " [16 1 9 19 5 3 3 3 3 3]\n", " [16 1 9 19 5 3 3 3 3 3]\n", " [16 1 9 19 5 3 3 3 3 3]]\n", "[[16 1 9 19 5 3 3 1 3 3]\n", " [18 8 5 13 5 3 3 3 3 3]\n", " [16 1 9 19 5 3 3 1 3 3]\n", " [16 1 9 19 5 3 3 1 3 3]\n", " [16 1 9 19 5 3 3 1 3 3]\n", " [16 1 9 19 5 3 3 1 3 3]]\n", "[[16 1 9 19 5 1 2 2 3 3]\n", " [18 8 5 13 5 3 3 3 3 3]\n", " [16 1 9 19 5 1 2 2 3 3]\n", " [16 1 9 19 5 1 2 2 3 3]\n", " [16 1 9 19 5 1 2 2 3 3]\n", " [16 1 9 19 5 1 2 2 3 3]]\n", "[[16 1 9 19 5 1 1 3 1 1]\n", " [18 1 11 5 4 2 1 3 2 3]\n", " [16 1 9 19 5 1 1 3 1 1]\n", " [16 1 9 19 5 1 1 3 1 1]\n", " [16 1 9 19 5 1 1 3 1 1]\n", " [16 1 9 19 5 1 1 3 1 1]]\n", "[[16 1 9 19 5 3 3 1 1 3]\n", " [18 1 11 5 4 3 3 3 3 3]\n", " [16 1 9 19 5 3 3 1 1 3]\n", " [16 1 9 19 5 3 3 1 1 3]\n", " [16 1 9 19 5 3 3 1 1 3]\n", " [16 1 9 19 5 3 3 1 1 3]]\n", "[[16 1 9 19 5 3 3 3 2 1]\n", " [18 1 11 5 4 3 3 3 2 3]\n", " [16 1 9 19 5 3 3 3 2 1]\n", " [16 1 9 19 5 3 3 3 2 1]\n", " [16 1 9 19 5 3 3 3 2 1]\n", " [16 1 9 19 5 3 3 3 2 1]]\n", "[[16 1 9 19 5 3 2 3 3 3]\n", " [18 8 5 13 5 1 3 3 2 3]\n", " [16 1 9 19 5 3 2 3 3 3]\n", " [16 1 9 19 5 3 2 3 3 3]\n", " [16 1 9 19 5 3 2 3 3 3]\n", " [16 1 9 19 5 3 2 3 3 3]]\n", "[[16 1 9 19 5 3 1 3 3 3]\n", " [18 8 5 13 5 3 3 3 3 3]\n", " [16 1 9 19 5 3 1 3 3 3]\n", " [16 1 9 19 5 3 1 3 3 3]\n", " [16 1 9 19 5 3 1 3 3 3]\n", " [16 1 9 19 5 3 1 3 3 3]]\n", "[[16 1 9 19 5 3 3 3 3 2]\n", " [18 8 5 13 5 3 3 1 1 2]\n", " [16 1 9 19 5 3 3 3 3 2]\n", " [16 1 9 19 5 3 3 3 3 2]\n", " [16 1 9 19 5 3 3 3 3 2]\n", " [16 1 9 19 5 3 3 3 3 2]]\n", "0\n" ] } ], "source": [ "env = gym_wordle.wordle.WordleEnv()\n", "\n", "for i in range(10):\n", " \n", " state, info = env.reset()\n", "\n", " done = False\n", "\n", " wins = 0\n", "\n", " while not done:\n", "\n", " action, _states = model.predict(state, deterministic=True)\n", "\n", " state, reward, done, truncated, info = env.step(action)\n", "\n", " if info[\"correct\"]:\n", " wins += 1\n", "\n", "print(wins)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 2 }