Fix caching for other APIs as well (#23)

Author: shengyfu

tokenizer_ts/package-lock.json

@@ -2,7 +2,7 @@
   "name": "@microsoft/tiktokenizer",
   "displayName": "tiktokenizer",
   "description": "Tokenizer for OpenAI large language models.",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "author": {
     "name": "Microsoft Corporation"
   },

tokenizer_ts/package.json

@@ -253,6 +253,7 @@ export class TikTokenizer {
         const bytes = this.textEncoder.encode(piece);
         const token = this.encoder!.get(uint8ArrayToString(bytes));
         if (token !== undefined) {
+          this.cache.set(piece, [token]);
           tokenCount++;
           if (tokenCount <= maxTokenCount) {
             encodeLength += piece.length;
@@ -397,6 +398,7 @@ export class TikTokenizer {
             const bytes = new TextEncoder().encode(piece);
             const token = this.encoder!.get(uint8ArrayToString(bytes));
             if (token !== undefined) {
+              this.cache.set(piece, [token]);
               tokenCount++;
               encodeLength += piece.length;
               tokenIds.push(token);