fixed the tokenizer

app/app.py

@@ -62,17 +62,8 @@ model_type = st.sidebar.selectbox('Model', (MODELS.keys()))
 @st.cache(suppress_st_warning=True, allow_output_mutation=True)
 def get_generator(model_name: str):
     st.write(f"Loading the GPT2 model {model_name}, please wait...")
-    special_tokens = AbstractDataset.special_tokens
     tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=hf_auth_token)
-    tokenizer.add_special_tokens(special_tokens)
-    config = AutoConfig.from_pretrained(model_name,
-                                        bos_token_id=tokenizer.bos_token_id,
-                                        eos_token_id=tokenizer.eos_token_id,
-                                        sep_token_id=tokenizer.sep_token_id,
-                                        pad_token_id=tokenizer.pad_token_id,
-                                        output_hidden_states=False,
-                                        use_auth_token=hf_auth_token)
-    model = GPT2LMHeadModel.from_pretrained(model_name, config=config, use_auth_token=hf_auth_token)
+    model = GPT2LMHeadModel.from_pretrained(model_name, pad_token_id=tokenizer.eos_token_id, use_auth_token=hf_auth_token)
     model.resize_token_embeddings(len(tokenizer))
     return model, tokenizer
 
@@ -81,24 +72,35 @@ def get_generator(model_name: str):
 # @st.cache(suppress_st_warning=True, hash_funcs={tokenizers.Tokenizer: id})
 def process(text_generator, tokenizer, title: str, keywords: str, text: str,
             max_length: int = 200, do_sample: bool = True, top_k: int = 50, top_p: float = 0.95,
-            temperature: float = 1.0, max_time: float = 120.0, seed=42, repetition_penalty=1.0):
+            temperature: float = 1.0, max_time: float = 120.0, seed=42, repetition_penalty=1.0,
+            penalty_alpha = 0.6):
     # st.write("Cache miss: process")
     set_seed(seed)
     if repetition_penalty == 0.0:
         min_penalty = 1.05
         max_penalty = 1.5
         repetition_penalty = max(min_penalty + (1.0 - temperature) * (max_penalty - min_penalty), 0.8)
-    print("title:", title)
-    print("keywords:", keywords)
+    # print("title:", title)
+    # print("keywords:", keywords)
     prompt = f"title: {title}\nkeywords: {keywords}\n{text}"
-    print("prompt: ", prompt)
+    # print("prompt: ", prompt)
 
     generated = torch.tensor(tokenizer.encode(prompt)).unsqueeze(0)
     # device = torch.device("cuda")
     # generated = generated.to(device)
 
+    print("do_sample:", do_sample)
+    print("penalty_alpha:", penalty_alpha)
+    print("max_length:", max_length)
+    print("top_k:", top_k)
+    print("top_p:", top_p)
+    print("temperature:", temperature)
+    print("max_time:", max_time)
+    print("repetition_penalty:", repetition_penalty)
+
     text_generator.eval()
     sample_outputs = text_generator.generate(generated,
+                                             penalty_alpha=penalty_alpha,
                                              do_sample=do_sample,
                                              min_length=200,
                                              max_length=max_length,
@@ -109,7 +111,7 @@ def process(text_generator, tokenizer, title: str, keywords: str, text: str,
                                              num_return_sequences=1
                                              )
     result = tokenizer.decode(sample_outputs[0], skip_special_tokens=True)
-    print(f"result: {result}")
+    # print(f"result: {result}")
     prefix_length = len(title) + len(keywords) + len("title: keywords: ") + 2
     result = result[prefix_length:]
     return result
@@ -173,6 +175,7 @@ if prompt_group_name in ["Indonesian Newspaper"]:
     top_k = 30
     top_p = 0.95
     repetition_penalty = 0.0
+    penalty_alpha = None
 
     if decoding_methods == "Beam Search":
         do_sample = False
@@ -191,7 +194,7 @@ if prompt_group_name in ["Indonesian Newspaper"]:
         )
     else:
         do_sample = False
-        repetition_penalty = 1.0
+        repetition_penalty = 1.1
         penalty_alpha = st.sidebar.number_input(
             "Penalty alpha",
             value=0.6,
@@ -237,11 +240,12 @@ if prompt_group_name in ["Indonesian Newspaper"]:
                              title=session_state.title,
                              keywords=session_state.keywords,
                              text=session_state.text, max_length=int(max_length),
-                             temperature=temperature, do_sample=do_sample,
+                             temperature=temperature, do_sample=do_sample, penalty_alpha=penalty_alpha,
                              top_k=int(top_k), top_p=float(top_p), seed=seed, repetition_penalty=repetition_penalty)
             time_end = time.time()
             time_diff = time_end - time_start
             # result = result[0]["generated_text"]
+            result = result[:result.find("title:")]
             st.write(result.replace("\n", "  \n"))
             st.text("Translation")
             translation = translate(result, "en", "id")