Update rpc.py

rpc.py

@@ -5,6 +5,7 @@ import keras_nlp
 
 import math
 import json
+import spacy
 from transformers import AutoTokenizer
 from tokenizers import AddedToken
 
@@ -23,6 +24,23 @@ print("vocab_size:", vocab_size)
 print("pad token id:", tokenizer.pad_token)
 
 
+nlp = spacy.load("en_core_web_lg")
+nlp.max_length = 2000000
+selected = {'NUM', 'PROPN'}
+alltoks = sorted(list(tokenizer.get_vocab().items()), key=lambda x:x[1])
+all_toks_text = "\n".join([t[0].replace("▁", "") for t in alltoks])
+doc = nlp(all_toks_text)
+carry_toks = set()
+i = 0
+for ii, token in enumerate(doc):
+    if str(token) in alltoks[i][0]: pass
+    else: i += 1
+    if str(token) in alltoks[i][0] and token.pos_ in selected and i > 100:
+        if (token.pos_ != "PROPN" or alltoks[i][0].replace("▁", "")[0].isupper()):
+            carry_toks.add(alltoks[i][1])
+print(len(carry_toks))
+
+
 # Masked Accuracy Metric
 def masked_accuracy(y_true, y_pred, padding_token=tokenizer.pad_token_id):
     y_true = tf.cast(y_true, tf.int32)