init

local_coh_ppl.py

@@ -211,6 +211,7 @@ class LocalCohPPL(evaluate.Measurement):
                 )
 
             # tokenize by sentence
+            sent_tok_lens = []
             for pred in batch_sents:
                 ss = pred.split("\n")
                 sslens = [len(tokenizer(y,add_special_tokens=False,padding=False).input_ids) for y in ss]