Update app.py

app.py

@@ -14,57 +14,153 @@ import base64
 import random
 from PIL import Image
 import gradio as gr
+openai.api_key ='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'
 
-def portal_scraping(show_more_count):
-    keyword = "ai"
-    url = 'https://www.cxnetwork.com/'
+def split_article(article_text):
+    words = article_text.split()    
+    total_words = len(words)
+    split_points = [total_words // 4, total_words // 2, (3 * total_words) // 4]
 
-    options = webdriver.ChromeOptions()
-    options.add_argument('--headless')
-    options.add_argument('--no-sandbox')
-    options.add_argument('--disable-dev-shm-usage')
-    
-    user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
-    options.add_argument(f"user-agent={user_agent}")
+    first_quarter = ' '.join(words[:split_points[0]])
+    second_quarter = ' '.join(words[split_points[0]:split_points[1]])
+    third_quarter = ' '.join(words[split_points[1]:split_points[2]])
+    fourth_quarter = ' '.join(words[split_points[2]:])
 
-    wd = webdriver.Chrome(options=options)
-    wd.get(url+"search/?q="+keyword)
+    return first_quarter, second_quarter, third_quarter, fourth_quarter
 
-    # Fungsi Untuk Menentukan Berapa Kali Menekan Tombol "Show More" Sesuai Dengan Keinginan User
-    def click_show_more():
-        wd.find_element(By.CSS_SELECTOR, 'body').send_keys(Keys.CONTROL, Keys.END)
-        time.sleep(3)
-        try:
-            elem = wd.find_element(By.CSS_SELECTOR, "span[class='semibold']")
-            elem.click()
-            time.sleep(15)
-        except:
-            print("error")
+def clean_scrap(artikel,url):
+    new_artikel = []
+    article = []
+    if len(artikel) > 1:
+        for art in artikel:
+            response = openai.ChatCompletion.create(
+                model ="gpt-3.5-turbo",
+                messages=[
+                    {"role": "system", "content": "You are a very professional article editor."},
+                    {"role": "user", "content": "I have a raw article that contains a lot of unnecessary data such as ads, website information, and article publishers, as well as links to other pages, and so on. Please clean up the article I provided so that only the article's content remains. \nThen, you should also summarize the article so that it does not exceed 5000 characters" + art + "\nDo not write any explanation and any pleasantries. Please use the following complete format to display the output: {the cleaned and summarized article's content}"}       
+                ],
+                temperature = 0.1
+            )
+            finish_reason = response['choices'][0]['finish_reason']
+            if finish_reason == 'length' or finish_reason == 'stop':
+                result = response['choices'][0]['message']['content']
+                new_artikel.append(result)
+            else:
+                with open(file_path, 'a') as file:
+                    file.write(url + '\n')
+                return None,None,None,None
+    else:
+         for art in artikel:
+            response = openai.ChatCompletion.create(
+                model ="gpt-3.5-turbo",
+                messages=[
+                    {"role": "system", "content": "You are a very professional article editor."},
+                    {"role": "user", "content": "I have a raw article that contains a lot of unnecessary data such as ads, website information, and article publishers, as well as links to other pages, and so on. Please clean up the article I provided so that only the article's content remains." + art + "\nDo not write any explanation and any pleasantries. Please use the following complete format to display the output: {the cleaned article's content}"}       
+                ],
+                temperature = 0.1
+            )
+            finish_reason = response['choices'][0]['finish_reason']
+            if finish_reason == 'length' or finish_reason == 'stop':
+                result = response['choices'][0]['message']['content']
+                new_artikel.append(result)
+            else:
+                with open(file_path, 'a') as file:
+                    file.write(url + '\n')
+                return None,None,None,None
+            
+    new_art = [' '.join(new_artikel)]
+    for art in new_art:
+        response = openai.ChatCompletion.create(
+            model ="gpt-3.5-turbo",
+            messages=[
+                {"role": "system", "content": "You are a very professional article editor and capable of generating compelling and professional article titles."},
+                {"role": "user", "content": "Paraphrase the above article to make it a well-written and easily understandable piece for humans, following the conventions of renowned articles. \nThen, You Must Generate a title that is appropriate for the article I provided. The title should be professional, similar to typical article titles and sound more natural for a human to read" + art + "\nDo not write any explanation and any pleasantries. Please use the following complete format to display the output: title:{title}, article: {new paraphrased article}"}       
+         ],
+            temperature = 0.1
+        )
+        finish_reason = response['choices'][0]['finish_reason']
+        if finish_reason == 'length' or finish_reason == 'stop':
+            result = response['choices'][0]['message']['content']
+            article.append(result)
+        else:
+            with open(file_path, 'a') as file:
+                file.write(url + '\n')
+            return None,None,None,None
 
-    for i in range(int(show_more_count)):
-        click_show_more()
+    content = article[0].split("\n")
+    title = content[0].replace('title:', '').strip()
+    response = openai.ChatCompletion.create(
+        model ="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": "You are a professional translator and rewriter"},
+            {"role": "user", "content": "Please translate and rewrite this sentence into Indonesian language with the following requirements: \n1. The sentence should be concise, compact, and clear. \n2. The sentence length should not exceed 50 characters. \n3. The sentences should be professional, similar to typical article titles and sound more natural for a human to read.:" +title+"\nDo not write any explanation and any pleasantries. Please use the following complete format to display the output: Judul:{hasil rewrite}"}        
+        ],
+        temperature = 0
+    )
+    judul = response['choices'][0]['message']['content']
+    judul = judul.replace("Judul:", '').strip()
+    judul = judul.replace("Title:", '').strip()
 
-    wd.find_element(By.CSS_SELECTOR, 'body').send_keys(Keys.CONTROL, Keys.END)
-    time.sleep(1)
+    contents = content[1:]
+    contents = [' '.join(contents).replace("article:", '').replace("Article:", '').strip()]
+    len_contents = len(contents[0])
+    if len_contents > 2000:
+        return title, judul, url, content
+    else:
+        with open(file_path, 'a') as file:
+            file.write(url + '\n')
+        return None,None,None,None
 
-    raw_html = wd.find_element(By.TAG_NAME, 'body').get_attribute('innerHTML')
-    wd.quit()
+def scrap_artikel(alamat):
+    artikel = []
+    link = alamat
+    for url in link:
+        if cek_url(url):
+            continue
+        else:
+            if len(artikel) >=1:
+                continue
+        options = webdriver.ChromeOptions()
+        options.add_argument('--headless')
+        options.add_argument('--no-sandbox')
+        options.add_argument('--disable-dev-shm-usage')
+        
+        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
+        options.add_argument(f"user-agent={user_agent}")
+    
+        wd = webdriver.Chrome(options=options)
+        wd.get(url+"search/?q="+keyword)
+    
+        wd.find_element(By.CSS_SELECTOR, 'body').send_keys(Keys.CONTROL, Keys.END)
+        time.sleep(1)
+    
+        raw_html = wd.find_element(By.TAG_NAME, 'body').get_attribute('innerHTML')
+        wd.quit()
+    
+        soup_html = BeautifulSoup(raw_html, "html.parser")
+        containers = soup_html.findAll('p')
 
-    soup_html = BeautifulSoup(raw_html, "html.parser")
-    # Membuat Variabel Yang digunakan Untuk Menyimpan URL Yang Berhasil discraping
-    portal = []
+        artikel =[]
+        for paragraph in containers:
+            artic=paragraph.get_text()
+            artikel.append(artic)
 
-    containers = soup_html.findAll('div', attrs={'class':'mt-2 ml-3 mr-3'})
-    for container in containers:
-        link = container.findAll('p')
-        for links in link:
-            url = links.findAll('a')
-            for urls in url:
-                href = urls.get('href')
-                portal.append(href)
+        paragraf = ' '.join(artikel)
+        len_paragraf = len(paragraf)
+        if len_paragraf >= 18000:
+            part1, part2, part3, part4 = split_article(paragraf)
+            artikels = [part1, part2, part3, part4]
+        else :
+            artikels = [paragraf]
+        len_artikel = len("".join(artikels))
+        if len_artikel > 1200 and len_artikel < 28000:
+            contents, url, title, judul = clean_scrap(artikels,url)
+            return title, judul, url, content
+        else:
+            with open(file_path, 'a') as file:
+                file.write(url + '\n')
+            return None,None,None,None
 
-    return portal
-        
 with gr.Blocks(theme = "soft", title="Article Writer") as article_generator:
     gr.Markdown(
             """
@@ -72,11 +168,14 @@ with gr.Blocks(theme = "soft", title="Article Writer") as article_generator:
             Buat artikel yang anda inginkan dengan mudah, cukup menuliskan keyword atau topik artikel yang ingin anda buat dan dalam sekejap artikel tersebut menjadi milik anda.
             """)
     with gr.Row():
-        inputs = gr.Textbox(placeholder="Keyword/Topik", show_label=False)
+        inputs = gr.Textbox(placeholder="Link Article", show_label=False)
     with gr.Row():
-        button_scrap = gr.Button("Scrap")
+        button_scrap = gr.Button("Scrap Article")
     with gr.Row():
-        outputs =  gr.Textbox("", label="URL")
-        button_scrap.click(fn=portal_scraping, inputs=inputs, outputs=outputs)
+        title =  gr.Textbox("", label="Title")
+        judul =  gr.Textbox("", label="Judul")
+        url =  gr.Textbox("", label="URL")
+        content =  gr.Textbox("", label="Content")
+        button_scrap.click(fn=scrap_artikel, inputs=inputs, outputs=[title,judul,url,content])
 if __name__ == "__main__":
     article_generator.launch(share=False)