add evaluation to eric's wordle solver (eval.py)

This commit is contained in:
Arthur Lu 2024-03-20 12:53:40 -07:00
parent 848d385482
commit c448e02512
2 changed files with 84 additions and 19 deletions

View File

@ -18,6 +18,16 @@ class AI:
self.reset()
def solve_eval(self, results_callback):
num_guesses = 0
while [len(e) for e in self.domains] != [1 for _ in range(self.num_letters)]:
num_guesses += 1
word = self.sample()
results = results_callback(word)
self.arc_consistency(word, results)
# print(num_guesses, word, results)
return num_guesses, word
def solve(self):
num_guesses = 0
while [len(e) for e in self.domains] != [1 for _ in range(self.num_letters)]:
@ -33,26 +43,29 @@ class AI:
print('-----------------------------------------------')
print(f'Guess #{num_guesses}/{self.num_guesses}: {word}')
print('-----------------------------------------------')
self.arc_consistency(word)
print(f'Performing arc consistency check on {word}...')
print(f'Specify 0 for completely nonexistent letter at the specified index, 1 for existent letter but incorrect index, and 2 for correct letter at correct index.')
results = []
# Collect results
for l in word:
while True:
result = input(f'{l}: ')
if result not in ['0', '1', '2']:
print('Incorrect option. Try again.')
continue
results.append(result)
break
print(results)
self.arc_consistency(word, results)
print(f'You did it! The word is {"".join([e[0] for e in self.domains])}')
return num_guesses
def arc_consistency(self, word):
print(f'Performing arc consistency check on {word}...')
print(f'Specify 0 for completely nonexistent letter at the specified index, 1 for existent letter but incorrect index, and 2 for correct letter at correct index.')
results = []
# Collect results
for l in word:
while True:
result = input(f'{l}: ')
if result not in ['0', '1', '2']:
print('Incorrect option. Try again.')
continue
results.append(result)
break
def arc_consistency(self, word, results):
self.possible_letters += [word[i] for i in range(len(word)) if results[i] == '1']
for i in range(len(word)):
@ -70,7 +83,6 @@ class AI:
if results[i] == '2':
self.domains[i] = [word[i]]
def reset(self):
self.domains = [list(string.ascii_lowercase) for _ in range(self.num_letters)]
self.possible_letters = []

53
eric_wordle/eval.py Normal file
View File

@ -0,0 +1,53 @@
import argparse
from ai import AI
import numpy as np
global solution
def result_callback(word):
global solution
result = ['0', '0', '0', '0', '0']
for i, letter in enumerate(word):
if solution[i] == word[i]:
result[i] = '2'
elif letter in solution:
result[i] = '1'
else:
pass
return result
def main(args):
global solution
if args.n is None:
raise Exception('Need to specify n (i.e. n = 1 for wordle, n = 4 for quordle, n = 16 for sedecordle).')
ai = AI(args.vocab_file)
total_guesses = 0
num_eval = args.num_eval
for i in range(num_eval):
idx = np.random.choice(range(len(ai.vocab)))
solution = ai.vocab[idx]
guesses, word = ai.solve_eval(results_callback=result_callback)
if word != solution:
total_guesses += 5
else:
total_guesses += guesses
ai.reset()
print(total_guesses / num_eval)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--n', dest='n', type=int, default=None)
parser.add_argument('--vocab_file', dest='vocab_file', type=str, default='wordle_words.txt')
parser.add_argument('--num_eval', dest="num_eval", type=int, default=1000)
args = parser.parse_args()
main(args)