fix cache miss problem (#22)

Author: sbatten

tokenizer_ts/package-lock.json

@@ -2,7 +2,7 @@
   "name": "@microsoft/tiktokenizer",
   "displayName": "tiktokenizer",
   "description": "Tokenizer for OpenAI large language models.",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "author": {
     "name": "Microsoft Corporation"
   },

tokenizer_ts/package.json

@@ -213,6 +213,7 @@ export class TikTokenizer {
         const token = this.encoder?.get(uint8ArrayToString(bytes));
         if (token !== undefined) {
           tokenIds.push(token);
+          this.cache.set(match[0], [token]);
         } else {
           const encodedTokens = bytePairEncode(bytes, this.encoder!);
           tokenIds.push(...encodedTokens);