Spaces:
Sleeping
Sleeping
File size: 3,944 Bytes
4d0ed30 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 |
# -*- coding: utf-8 -*-
"""document_scrapped.ipynb
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/1cVGt7jq8uw5FYIwWOlUTAFbdVhPkU1FJ
"""
!pip install -r requirements.txt
!pip install python-docx
!pip install docx
!pip install PyMuPDF
!pip install python-pptx
from bs4 import BeautifulSoup
import requests
import json
import io
import fitz
from pptx import Presentation
import chardet
from docx import Document
import pandas as pd
def downl(url):
try:
rq = requests.get(url)
if rq.status_code != 200:
return None
bs = BeautifulSoup(rq.text, features='lxml')
lis = bs.find_all('ul', class_='dropdown-menu')[-1].find_all('li')
link = lis[-1].find('a').get('href')
return link
except Exception as e:
return None
def excel(link):
try:
ls = downl(link)
response = requests.get(ls)
if response.status_code == 200:
file_content = response.content
df = pd.read_excel(file_content)
if df.shape[0] > 50:
sample_size = 50
sample_df = df.sample(n=sample_size, random_state=42)
json_data = sample_df.to_json(orient='records')
js = json.loads(json_data)
return js
else:
print("Failed to download file")
except Exception as e:
return None
def csv(link):
try:
ls = downl(link)
print(ls)
response = requests.get(ls)
if response.status_code == 200:
file_content = response.content
detected_encoding = chardet.detect(file_content)['encoding']
df = pd.read_csv(io.BytesIO(file_content), encoding=detected_encoding, sep=';')
if df.empty:
print("The DataFrame is empty.")
return None
if df.shape[0] > 50:
sample_size = 50
sample_df = df.sample(n=sample_size, random_state=42)
else:
sample_df = df
json_data = sample_df.to_json(orient='records')
js = json.loads(json_data)
return js
except Exception as e:
return None
def docx(url):
try:
ls = downl(url)
# Download the .docx file
response = requests.get(ls)
response.raise_for_status() # Ensure we notice bad responses
# Read the .docx file
file_stream = io.BytesIO(response.content)
doc = Document(file_stream)
# Extract text
full_text = []
for para in doc.paragraphs:
full_text.append(para.text)
return '\n'.join(full_text)
except Exception as e:
print(f"An error occurred: {e}")
return None
def pdf(url):
try:
ls = downl(url)
# Download the PDF file
response = requests.get(ls)
response.raise_for_status() # Ensure we notice bad responses
# Read the PDF file
file_stream = io.BytesIO(response.content)
pdf_document = fitz.open(stream=file_stream, filetype='pdf')
# Extract text
full_text = []
for page_num in range(len(pdf_document)):
page = pdf_document.load_page(page_num)
full_text.append(page.get_text())
return '\n'.join(full_text)
except Exception as e:
print(f"An error occurred: {e}")
return None
def pptx(url):
try:
ls = downl(url)
response = requests.get(ls)
response.raise_for_status()
# Read the .pptx file
file_stream = io.BytesIO(response.content)
presentation = Presentation(file_stream)
# Extract text
full_text = []
for slide in presentation.slides:
for shape in slide.shapes:
if hasattr(shape, "text"):
full_text.append(shape.text)
return '\n'.join(full_text)
except Exception as e:
print(f"An error occurred: {e}")
return None |