Update app.py
Browse files
app.py
CHANGED
@@ -28,6 +28,15 @@ basicConfig(
|
|
28 |
filemode='a'
|
29 |
)
|
30 |
logger = getLogger('proxy_collector')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
31 |
|
32 |
is_first_run = True
|
33 |
|
@@ -40,6 +49,7 @@ countries_list = ['US', 'CA', 'FR', 'FI', 'HR', 'ME', 'CH', 'SE', 'EE', 'DE', 'G
|
|
40 |
|
41 |
|
42 |
def create_json_from_proxies(proxy_lines: list[str], filename: Path):
|
|
|
43 |
countries = set()
|
44 |
proxies = []
|
45 |
|
@@ -72,11 +82,12 @@ async def collect_proxies(proxies_queue: Queue):
|
|
72 |
if proxy is None:
|
73 |
break
|
74 |
proxies_list.append(f'{proxy}')
|
75 |
-
print(f"\rtotal proxies: {len(proxies_list)}", end='')
|
76 |
return proxies_list
|
77 |
|
78 |
|
79 |
async def sort_proxies_and_merge(files: list[Path], output_file: Path):
|
|
|
80 |
all_countries = set()
|
81 |
proxies_by_type = {}
|
82 |
for file in files:
|
@@ -116,7 +127,7 @@ async def find_proxies_by_type(proxy_type: Literal['HTTP', 'HTTPS', 'SOCKS5'], o
|
|
116 |
await broker.find(types=[proxy_type], countries=countries_list, limit=0)
|
117 |
await stop_task
|
118 |
proxies_list = await collect_proxies(proxies_queue)
|
119 |
-
saved_proxy =
|
120 |
logger.info(f'завершён сбор прокси {proxy_type}')
|
121 |
return saved_proxy
|
122 |
|
@@ -136,6 +147,7 @@ async def find_proxies():
|
|
136 |
|
137 |
|
138 |
scheduler.add_job(find_proxies, 'interval', max_instances=1, minutes=60)
|
|
|
139 |
|
140 |
|
141 |
@asynccontextmanager
|
|
|
28 |
filemode='a'
|
29 |
)
|
30 |
logger = getLogger('proxy_collector')
|
31 |
+
logger.info('запуск...')
|
32 |
+
|
33 |
+
|
34 |
+
def delete_logs():
|
35 |
+
try:
|
36 |
+
logfile.unlink()
|
37 |
+
except Exception as e:
|
38 |
+
logger.error(f'ошибка при удалении логов: {e}')
|
39 |
+
|
40 |
|
41 |
is_first_run = True
|
42 |
|
|
|
49 |
|
50 |
|
51 |
def create_json_from_proxies(proxy_lines: list[str], filename: Path):
|
52 |
+
logger.info('сохранение файла прокси')
|
53 |
countries = set()
|
54 |
proxies = []
|
55 |
|
|
|
82 |
if proxy is None:
|
83 |
break
|
84 |
proxies_list.append(f'{proxy}')
|
85 |
+
# print(f"\rtotal proxies: {len(proxies_list)}", end='')
|
86 |
return proxies_list
|
87 |
|
88 |
|
89 |
async def sort_proxies_and_merge(files: list[Path], output_file: Path):
|
90 |
+
logger.info('объединение файлов прокси')
|
91 |
all_countries = set()
|
92 |
proxies_by_type = {}
|
93 |
for file in files:
|
|
|
127 |
await broker.find(types=[proxy_type], countries=countries_list, limit=0)
|
128 |
await stop_task
|
129 |
proxies_list = await collect_proxies(proxies_queue)
|
130 |
+
saved_proxy = create_json_from_proxies(proxies_list, output_json_file)
|
131 |
logger.info(f'завершён сбор прокси {proxy_type}')
|
132 |
return saved_proxy
|
133 |
|
|
|
147 |
|
148 |
|
149 |
scheduler.add_job(find_proxies, 'interval', max_instances=1, minutes=60)
|
150 |
+
scheduler.add_job(delete_logs, 'interval', max_instances=1, minutes=1440)
|
151 |
|
152 |
|
153 |
@asynccontextmanager
|