This repository has been archived by the owner on Sep 14, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parse.py
122 lines (118 loc) · 5.58 KB
/
parse.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#!/usr/bin/python3
from bs4 import BeautifulSoup
import requests as req
from time import sleep
import os, subprocess
from datetime import datetime
from PIL import Image, UnidentifiedImageError
from methods import Methods
from config import tmp_dir, vk_info
import tg_methods as Methods_tg
headers = {
'User-Agent': 'ShawelBot/Parser_v2.1'
}
for_parse = {
'rasp': 0,
#'zvonki': 2
}
def check(dir_):
for name in sorted(os.listdir(f'{tmp_dir}/parse/{dir_}')):
try: im = Image.open(f'{tmp_dir}/parse/{dir_}/{name}')
except UnidentifiedImageError: continue
white = 0
for n in im.getdata():
if((n[0] == 255 and n[1] == 255 and n[2] == 255) or n == (255, 0)):
white += 1
percent = white/(im.size[0]*im.size[1])
if(percent >= 0.995):
os.remove(f'{tmp_dir}/parse/{dir_}/{name}')
def parse(for_parse, manual = False):
if(os.path.isdir(tmp_dir + '/parse') != True):
os.makedirs(tmp_dir + '/parse')
if(os.path.isfile(tmp_dir + '/parse/parser.tmp')):
return 10
Mysql = Methods.Mysql()
# Methods.log("Parser", "Парсер выполняет проверку...")
page = req.get("https://engschool9.ru/content/raspisanie.html", headers=headers)
if(page.status_code != 200):
Methods.log("Parser", f"Получен неожиданный http код {self.page.status_code}")
return self.page.status_code
now = datetime.now()
date = now.strftime("%H:%M %d.%m.%Y")
page = BeautifulSoup(page.text, 'html.parser')
iframes = page.findAll("iframe")
with open(tmp_dir + '/parse/parser.tmp', 'w') as f:
f.write('')
try:
for prefix, cnt in for_parse.items():
curdir = tmp_dir+"/parse/"+prefix
if(os.path.isdir(curdir) == False):
os.makedirs(curdir)
res = Methods.setting_get(f'{prefix}_last')
if(res == iframes[cnt]['src']):
if(manual):
return 11
continue
with open(f"{tmp_dir}/parse/{prefix}.pdf", "wb") as f:
r = req.get(iframes[cnt]['src'], stream=True)
if(r.status_code != 200):
Methods.log("WARN", f"Парсер [{prefix}] не может загрузить файл {iframes[cnt]['src']}. Код: {r.status_code}")
continue
for chunk in r.iter_content(chunk_size = 1024):
if(chunk): f.write(chunk)
subprocess.Popen(["convert", "-colorspace", "RGB", "-density", "200", f"{tmp_dir}/parse/{prefix}.pdf", f"{curdir}/out.png"], stdout=subprocess.DEVNULL).wait()
files_count = len(os.listdir(curdir))
check(prefix)
attach = []
for img in sorted(os.listdir(curdir)):
attach.append(Methods.upload_img_post(f"{curdir}/{img}"))
txt = 'Зафиксировано обновление\nДля отписки используйте команду \'/рассылка\''
# post_pre = Methods.setting_get('post_pre')
# if(post_pre != '' and post_pre != None):
# Methods.wall_del(post_pre)
txt1 = f"Файл: {iframes[cnt]['src'].split('/')[-1]}\nВремя: {date}\nИзображений: {len(attach)}/{files_count}"
post_id = Methods.wall_post(txt1,attachments=attach)['post_id']
if(for_parse[prefix] == 2):
Methods.wall_pin(post_id)
Methods.setting_set(f'{prefix}_post', post_id)
attach = f"wall-{vk_info['groupid']}_{post_id}"
i = 0; i_limit = 50
users = Mysql.query("SELECT vkid FROM `users` WHERE subscribe = 1 LIMIT %s, %s", (i, i_limit), fetch="all")
while users:
send_users = []
for user in users:
send_users.append(str(user['vkid']))
send_users = ",".join(send_users)
Methods.mass_send(send_users, message=txt, attachment=attach, keyboard=Methods.construct_keyboard(b1=Methods.make_button(label="Отписаться", color="negative"), one_time="true"), intent="non_promo_newsletter")
i += i_limit
sleep(.3)
users = Mysql.query("SELECT vkid FROM `users` WHERE subscribe = 1 LIMIT %s, %s", (i, i_limit), fetch="all")
i = 0
chats = Mysql.query("SELECT id FROM `chats` WHERE subscribe = 1 LIMIT %s, %s", (i, i_limit), fetch="all")
while chats:
send_chats = []
for chat in chats:
send_chats.append(str(chat['id']))
send_chats = ",".join(send_chats)
Methods.mass_send(send_chats, message=txt, attachment=attach)
i += i_limit
sleep(.3)
chats = Mysql.query("SELECT id FROM `chats` WHERE subscribe = 1 LIMIT %s, %s", (i, i_limit), fetch="all")
Methods.setting_set(f'{prefix}_last', iframes[cnt]['src'])
Methods_tg.send_to_users(prefix, curdir, txt1)
Methods.log(f"{prefix.title()}Parser", "Зафиксировано обновление")
finally:
for root, dirs, files in os.walk(tmp_dir+"/parse", topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
Mysql.close()
def run():
while True:
if(datetime.now().minute % 10 == 0):
try:
parse(for_parse)
except Exception as e:
Methods.log("ERROR", e)
sleep(60)