init

local_coh_ppl.py

@@ -218,7 +218,8 @@ class LocalCohPPL(evaluate.Measurement):
                 offset = int(add_start_token)
                 sspos = [offset]
                 for sslen in sslens:
-                    offset = min(offset + sslen,511)
+                    offset = min(offset + sslen,511 + int(add_start_token))
+                    if offset == sspos[-1]: break # reached length limit
                     sspos.append(offset)
                 sent_tok_lens.append(sspos)