add evaluation to eric's wordle solver (eval.py)

eric_wordle/ai.py

@@ -17,7 +17,17 @@ class AI:
         self.possible_letters = None
 
         self.reset()
-
+        
+    def solve_eval(self, results_callback):
+        num_guesses = 0
+        while [len(e) for e in self.domains] != [1 for _ in range(self.num_letters)]:
+            num_guesses += 1
+            word = self.sample()
+            results = results_callback(word)
+            self.arc_consistency(word, results)
+            # print(num_guesses, word, results)
+        return num_guesses, word
+    
     def solve(self):
         num_guesses = 0
         while [len(e) for e in self.domains] != [1 for _ in range(self.num_letters)]:
@@ -33,26 +43,29 @@ class AI:
             print('-----------------------------------------------')
             print(f'Guess #{num_guesses}/{self.num_guesses}: {word}')
             print('-----------------------------------------------')
-            self.arc_consistency(word)
+
+            print(f'Performing arc consistency check on {word}...')
+            print(f'Specify 0 for completely nonexistent letter at the specified index, 1 for existent letter but incorrect index, and 2 for correct letter at correct index.')
+            results = []
+
+            # Collect results
+            for l in word:
+                while True:
+                    result = input(f'{l}: ')
+                    if result not in ['0', '1', '2']:
+                        print('Incorrect option. Try again.')
+                        continue
+                    results.append(result)
+                    break
+
+            print(results)
+
+            self.arc_consistency(word, results)
 
         print(f'You did it! The word is {"".join([e[0] for e in self.domains])}')
+        return num_guesses
 
-
-    def arc_consistency(self, word):
-        print(f'Performing arc consistency check on {word}...')
-        print(f'Specify 0 for completely nonexistent letter at the specified index, 1 for existent letter but incorrect index, and 2 for correct letter at correct index.')
-        results = []
-
-        # Collect results
-        for l in word:
-            while True:
-                result = input(f'{l}: ')
-                if result not in ['0', '1', '2']:
-                    print('Incorrect option. Try again.')
-                    continue
-                results.append(result)
-                break
-
+    def arc_consistency(self, word, results):
         self.possible_letters += [word[i] for i in range(len(word)) if results[i] == '1']
 
         for i in range(len(word)):
@@ -70,7 +83,6 @@ class AI:
             if results[i] == '2':
                 self.domains[i] = [word[i]]
 
-
     def reset(self):
         self.domains = [list(string.ascii_lowercase) for _ in range(self.num_letters)]
         self.possible_letters = []

eric_wordle/eval.py

@@ -0,0 +1,53 @@
+import argparse
+from ai import AI
+import numpy as np
+
+global solution
+
+def result_callback(word):
+
+    global solution
+
+    result = ['0', '0', '0', '0', '0']
+
+    for i, letter in enumerate(word):
+
+        if solution[i] == word[i]:
+            result[i] = '2'
+        elif letter in solution:
+            result[i] = '1'
+        else:
+            pass
+
+    return result
+
+def main(args):
+    global solution 
+
+    if args.n is None:
+        raise Exception('Need to specify n (i.e. n = 1 for wordle, n = 4 for quordle, n = 16 for sedecordle).')
+
+    ai = AI(args.vocab_file)
+
+    total_guesses = 0
+    num_eval = args.num_eval
+
+    for i in range(num_eval):
+        idx = np.random.choice(range(len(ai.vocab)))
+        solution = ai.vocab[idx]
+        guesses, word = ai.solve_eval(results_callback=result_callback)
+        if word != solution:
+            total_guesses += 5
+        else:
+            total_guesses += guesses
+        ai.reset()
+
+    print(total_guesses / num_eval)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--n', dest='n', type=int, default=None)
+    parser.add_argument('--vocab_file', dest='vocab_file', type=str, default='wordle_words.txt')
+    parser.add_argument('--num_eval', dest="num_eval", type=int, default=1000)
+    args = parser.parse_args()
+    main(args)