Spaces:

bigcode
/

tokenizer-testing

Build error

cakiki commited on Nov 13, 2022

Commit

67fe836

1 Parent(s): 5131ece

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -4,7 +4,7 @@ from transformers import AutoTokenizer
 st.set_page_config(page_title="BigCode Tokenizer", page_icon='👩‍💻', layout="wide")
-tokenizer = AutoTokenizer.from_pretrained('bigcode/tokenizer', subfolder="digit-punctuation-bytelevel-bpe-py-js-java-50k")
 visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
 text = st.text_area(label="", placeholder="Text to tokenize")

 st.set_page_config(page_title="BigCode Tokenizer", page_icon='👩‍💻', layout="wide")
+tokenizer = AutoTokenizer.from_pretrained('bigcode/tokenizer', subfolder="digit-custom_punctuation-bytelevel-bpe-py-js-java-50k")
 visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
 text = st.text_area(label="", placeholder="Text to tokenize")