Update human_text_detect.py

human_text_detect.py

@@ -94,7 +94,6 @@ def detect_human_text(model_name, topic, text):
     max_tokens_per_sentence = 100
 
     cache_dir = f"/tmp/cacheHuggingface/{model_name}"
-    os.environ["TRANSFORMERS_CACHE"] = cache_dir
     
     print('Create dir')
     # Use a writable directory inside the Hugging Face Space 
@@ -104,7 +103,13 @@ def detect_human_text(model_name, topic, text):
     # Init model
     print('Init tokenizer')
     lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
-    tokenizer = AutoTokenizer.from_pretrained(cache_dir) # lm_name, cache_dir=cache_dir
+    tokenizer = AutoTokenizer.from_pretrained(lm_name, cache_dir=cache_dir)
+
+    print("Save tokenizer")
+    tokenizer.save_pretrained(cache_dir)
+
+    print("Checking saved tokenizer files in:", cache_dir)
+    print(os.listdir(cache_dir))
     
     print('Init model')
     model = AutoModelForCausalLM.from_pretrained(lm_name) #, cache_dir=cache_dir