Update human_text_detect.py

human_text_detect.py

@@ -87,7 +87,7 @@ def detect_human_text(model_name, topic, text):
         df_null = df_null[df_null.num > 1]
     
     # Get survival function
-        print('Get survival function')
+    print('Get survival function')
     pval_functions = get_survival_function(df_null, G=43)
 
     min_tokens_per_sentence = 10
@@ -95,30 +95,19 @@ def detect_human_text(model_name, topic, text):
 
     cache_dir = f"/tmp/cacheHuggingface/{model_name}"
     
-    print('Create dir')
-    # Use a writable directory inside the Hugging Face Space 
-    # os.makedirs("/tmp/cacheHuggingface/PHI2", exist_ok=True)
-    # os.makedirs("/tmp/cacheHuggingface/GPT2XL", exist_ok=True)
-    
     # Init model
     print('Init tokenizer')
     lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
     tokenizer = AutoTokenizer.from_pretrained(lm_name, cache_dir=cache_dir)
-
-    print("Before saved tokenizer files in:", cache_dir)
-    print(os.listdir(cache_dir))
     
-    print("Save tokenizer")
-    tokenizer.save_pretrained(cache_dir)
-
-    print("Checking saved tokenizer files in:", cache_dir)
-    print(os.listdir(cache_dir))
+    # print("Save tokenizer")
+    # tokenizer.save_pretrained(cache_dir)
     
     print('Init model')
-    model = AutoModelForCausalLM.from_pretrained(lm_name) #, cache_dir=cache_dir
+    model = AutoModelForCausalLM.from_pretrained(lm_name, cache_dir=cache_dir)
 
-    print("Save model")
-    model.save_pretrained(cache_dir)
+    # print("Save model")
+    # model.save_pretrained(cache_dir)
     
     print('Init PerplexityEvaluator')
     sentence_detector = PerplexityEvaluator(model, tokenizer)