Update app.py
Browse files
app.py
CHANGED
@@ -97,76 +97,6 @@ def download_gitlab_repo():
|
|
97 |
print("Upload complete")
|
98 |
|
99 |
|
100 |
-
def get_all_files_in_folder(temp_dir, folder_path):
|
101 |
-
|
102 |
-
all_files = []
|
103 |
-
target_dir = os.path.join(temp_dir, folder_path)
|
104 |
-
|
105 |
-
for root, dirs, files in os.walk(target_dir):
|
106 |
-
print(f"Files in current directory ({root}): {files}")
|
107 |
-
for file in files:
|
108 |
-
print(f"Processing file: {file}")
|
109 |
-
all_files.append(os.path.join(root, file))
|
110 |
-
|
111 |
-
return all_files
|
112 |
-
|
113 |
-
def get_file(temp_dir, file_path):
|
114 |
-
full_path = os.path.join(temp_dir, file_path)
|
115 |
-
return full_path
|
116 |
-
|
117 |
-
|
118 |
-
def process_directory(directory, folder_paths, file_paths):
|
119 |
-
all_texts = []
|
120 |
-
file_references = []
|
121 |
-
|
122 |
-
zip_filename = next((file for file in os.listdir(directory) if file.endswith('.zip')), None)
|
123 |
-
zip_file_path = os.path.join(directory, zip_filename)
|
124 |
-
|
125 |
-
with tempfile.TemporaryDirectory() as tmpdirname:
|
126 |
-
# Unzip the file into the temporary directory
|
127 |
-
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
|
128 |
-
zip_ref.extractall(tmpdirname)
|
129 |
-
|
130 |
-
files = []
|
131 |
-
print("tmpdirname: " , tmpdirname)
|
132 |
-
unzipped_root = os.listdir(tmpdirname)
|
133 |
-
print("unzipped_root ", unzipped_root)
|
134 |
-
|
135 |
-
tmpsubdirpath= os.path.join(tmpdirname, unzipped_root[0])
|
136 |
-
print("tempsubdirpath: ", tmpsubdirpath)
|
137 |
-
|
138 |
-
if folder_paths:
|
139 |
-
for folder_path in folder_paths:
|
140 |
-
files += get_all_files_in_folder(tmpsubdirpath, folder_path)
|
141 |
-
if file_paths:
|
142 |
-
files += [get_file(tmpsubdirpath, file_path) for file_path in file_paths]
|
143 |
-
|
144 |
-
|
145 |
-
print(f"Total number of files: {len(files)}")
|
146 |
-
|
147 |
-
for file_path in files:
|
148 |
-
# print("111111111:", file_path)
|
149 |
-
file_ext = os.path.splitext(file_path)[1]
|
150 |
-
# print("222222222:", file_ext)
|
151 |
-
if os.path.getsize(file_path) == 0:
|
152 |
-
print(f"Skipping an empty file: {file_path}")
|
153 |
-
continue
|
154 |
-
|
155 |
-
with open(file_path, 'rb') as f:
|
156 |
-
if file_ext in ['.rst', '.py']:
|
157 |
-
text = f.read().decode('utf-8')
|
158 |
-
|
159 |
-
all_texts.append(text)
|
160 |
-
print("Filepaths brother:", file_path)
|
161 |
-
relative_path = os.path.relpath(file_path, tmpsubdirpath)
|
162 |
-
print("Relative Filepaths brother:", relative_path)
|
163 |
-
file_references.append(relative_path)
|
164 |
-
|
165 |
-
return all_texts, file_references
|
166 |
-
|
167 |
-
|
168 |
-
|
169 |
-
|
170 |
def split_python_code_into_chunks(texts, file_paths):
|
171 |
chunks = []
|
172 |
for text, file_path in zip(texts, file_paths):
|
|
|
97 |
print("Upload complete")
|
98 |
|
99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
100 |
def split_python_code_into_chunks(texts, file_paths):
|
101 |
chunks = []
|
102 |
for text, file_path in zip(texts, file_paths):
|