From aa93dfefb977bbacc82f68baf16ece51d9f33c03 Mon Sep 17 00:00:00 2001 From: Artem Lavrenov Date: Sat, 12 Dec 2020 14:40:25 +0300 Subject: [PATCH] fix onlime --- .gitignore | 1 + changelist.md | 10 ++++- plugin/make_stock_stat.py | 42 +++++++++++++++++ plugin/onlime.py | 77 ++++++++++--------------------- plugin/settings.py | 2 + plugin/stock.py | 30 ++++++++----- plugin/test3.py | 95 ++++++++------------------------------- plugin/test4.py | 86 ++++++++++++++++++++++++++--------- 8 files changed, 180 insertions(+), 163 deletions(-) create mode 100644 plugin/make_stock_stat.py diff --git a/.gitignore b/.gitignore index 583d4db..9e43b7a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ webdriver/* BalanceHistory.sqlite balance.html plugin/personalsetting.py +plugin/debug.log mbplugin*.zip python-*.zip dllplugin diff --git a/changelist.md b/changelist.md index a9375a9..533375f 100644 --- a/changelist.md +++ b/changelist.md @@ -145,6 +145,14 @@ __Автономная версия__: Исправление ошибки показа браузера (багрепорт comr) ## mbplugin 0.99.26 (03.11.20) Show history for standalone version -В этой версии изменения коснулись только Stabdalone версии. +В этой версии изменения коснулись только Standalone версии. В standalone версии можно включить показ истории по балансу (RealAverageDays=30), все остальные тонкие настройки см как обычно в settings.py Помните, что чем больше дней, тем дольше строится страница с балансом + +## mbplugin 0.99.26 (12.12.20) Fix onlime +В плагине mts2 добавлено получение баланса кэшбэка в переменную Balance2 +Исправлена работа провайдера Onlime (my.rt.ru ростелеком москва) - теперь он тоже на хроме +Плагин stock теперь может забирать цену ETF(фонды) с moex.com +Плагин stock может сохранять детальный лог по ценным бумагам чтобы по нему строить развернутую аналитику (для подготовки аналитики из лога скрипт make_stock_stat.py) +Fix. В плагине megafon были проблемы когда имя было с кавычками (так бывает когда телефон на организацию) +a1by добавлено закрытие баннера diff --git a/plugin/make_stock_stat.py b/plugin/make_stock_stat.py new file mode 100644 index 0000000..31d218b --- /dev/null +++ b/plugin/make_stock_stat.py @@ -0,0 +1,42 @@ +import sys, win32com.client +open(sys.argv[1]) +data = [line.split('\t') for line in open(sys.argv[1]).read().splitlines() if '\t' in line] +''' +all_date = {d:0. for d in list(set([i[0] for i in data]))} +all_stock = list(set([i[1] for i in data])) +table={s:all_date.copy() for s in all_stock} +for line in data: + table[line[1]][line[0]]=line[6] +#print(table) +print('\t'+'\t'.join(all_date.keys())) +for stock,line in table.items(): + print(stock+'\t'+'\t'.join([str(i) for i in line.values()])) +''' +all_stock = {i[1]:0. for i in data} +all_date = list({i[0]:None for i in data}) +table={d:all_stock.copy() for d in all_date} +for line in data: + table[line[0]][line[1]]=line[6] +#print(table) +print('\t'+'\t'.join(all_stock.keys())) +for date,line in table.items(): + print(date+'\t'+'\t'.join([str(i).replace('.',',') for i in line.values()])) + +xlsapp = win32com.client.gencache.EnsureDispatch ("Excel.Application") # Если создавать COM объект так то Sheets.Add отрабатывает правильно +xlsapp.Visible = True # Окно EXCEL видимое +xlsapp.DisplayAlerts = False # Подавить ошибки +wb_res=xlsapp.Workbooks.Add() + + +rng = wb_res.Sheets[1].Range(wb_res.Sheets[1].Cells(1,2),wb_res.Sheets[1].Cells(1,len(all_stock.keys())+1)) +rng.Value=tuple(all_stock.keys()) +#wb_res.Sheets[1].Range('A1:D1').Value=('H1','H2','H3','H4',) +wb_res.Sheets[1].Select() +xlsapp.ActiveWindow.SplitColumn = 0 +xlsapp.ActiveWindow.SplitRow = 1 +xlsapp.ActiveWindow.FreezePanes = True +wb_res.Sheets[1].Columns.AutoFilter(Field=1) +#wb_res.Sheets[1].Columns(wb_res.Sheets[1].Columns(1),wb_res.Sheets[1].Columns(len(all_stock.keys())+2)).AutoFilter(Field=1) +for num,(date,line) in enumerate(table.items()): + wb_res.Sheets[1].Cells(num+2,1).Value = date + wb_res.Sheets[1].Range(wb_res.Sheets[1].Cells(num+2,2),wb_res.Sheets[1].Cells(num+2,len(all_stock.keys())+1)).Value=tuple(line.values()) diff --git a/plugin/onlime.py b/plugin/onlime.py index 83e05c1..20549b4 100644 --- a/plugin/onlime.py +++ b/plugin/onlime.py @@ -1,63 +1,34 @@ # -*- coding: utf8 -*- ''' Автор ArtyLa ''' -import os, sys, re, logging -import requests -import store - -re_csrf = r'(?usi)_csrf_token.*?value="(\w+?)"' +import pyppeteeradd as pa icon = '789C73F235636100033320D600620128666450804840E5A905989999F1CAB359A833B0DBEB90AC9F595A88817F423283D895490C5C31F644EB67E46065E0CEF264103DD3CB20B2AF9924FDEC2EFA0CC23B1B1844CFF531F0E4FB3230097013AD9F7F6A1A582DFF943406663911885BD85888D62F7AAC8381B72E9C81919111A2174833B2B1E2D40F5387AC9FA7D08F4150408841008841348FA0205C3F0F0F2F032F2F1F0307072703171737033FBF2056FD1CEC1C0C6C6CEC0C3CDCBC0C2C9C1C70FDDCDC3C0CEC40391066656503D3C8FA450EB63288EC6D6260773744B811C9FF30F7E2F23FB38C3003FFC414B07A8179B90C2CAA5224851F0CB0596930086DAA6610BB38011C9EA4EA0703166606AE782706D1135DE4E9870226113E06BE966806CE683BB2F4C300232BEE4C4B8C7EBC6603E305008A3A3F17' -def get_balance(login, password, storename=None): - logging.info(f'start get_balance {login}') - result = {} - baseurl = 'https://my.rt.ru/' - cabinet_url = 'https://my.rt.ru/json/cabinet/' - headers = {} - session = store.Session(storename, headers=headers) - response3 = session.post(cabinet_url, data={}) - # !!! Хоть и возвращает json но 'content-type' - text/html - if 'accountInfo' in response3.text: - logging.info(f'Already logoned {login}') - else: - # Логинимся - logging.info(f'relogon {login}') - session.drop_and_create() - response1 = session.get(baseurl) - if response1.status_code != 200: - raise RuntimeError(f'GET Login page {baseurl} error: status_code {response1.status_code}') - data = {'_csrf_token': re.findall(re_csrf, response1.text), - 'login': login, - 'password': password,} - login_url = 'https://my.rt.ru/session/checklogin/' - response2 = session.post(login_url, data=data) - if response2.status_code != 200: - raise RuntimeError(f'POST Login page {login_url} error: status_code {response2.status_code}') - response3 = session.post(cabinet_url, data={}) - - if 'accountInfo' not in response3.text: - raise RuntimeError(f'Balance (accountInfo) not found on {cabinet_url}') +class sipnet_over_puppeteer(pa.balance_over_puppeteer): + async def async_main(self): + await self.do_logon( + url='https://my.rt.ru/', + user_selectors={'chk_lk_page_js': "document.querySelector('div.lk-login input[type=password]') == null", + 'chk_login_page_js': "document.querySelector('div.lk-login input[type=password]') !== null", + 'login_clear_js': "document.querySelector('div.lk-login input[name=auth_login]').value=''", + 'password_clear_js': "document.querySelector('div.lk-login input[type=password]').value=''", + 'login_selector': 'div.lk-login input[name=auth_login]', + 'password_selector': 'div.lk-login input[type=password]', + 'submit_selector': 'div.lk-login input[type=submit]' + }) + # Здесь мы берем данные с загружаемой страницы + await self.wait_params(params=[ + {'name': 'Balance', 'url_tag': ['json/cabinet/'], 'jsformula': "data.accountInfo.balance"}, + {'name': 'Balance2', 'url_tag': ['json/cabinet/'], 'jsformula': "data.bonusAccount.points"}, + {'name': 'licSchet', 'url_tag': ['json/cabinet/'], 'jsformula': "data.accountInfo.AccountID"}, + {'name': 'Expired', 'url_tag': ['json/cabinet/'], 'jsformula': "data.accountInfo.daysToLock"}, + {'name': 'anyString', 'url_tag': ['json/cabinet/'], 'jsformula': "data.bonusAccount.tier"}, + ]) - result['Balance'] = response3.json().get('accountInfo', {})['balance'] - - try: - result['Balance2'] = response3.json().get('bonusAccount',{})['points'] - except Exception: - logging.info(f'Not found bonusAccount') - try: - result['licSchet'] = response3.json().get('accountInfo', {})['AccountID'] - result['Expired'] = response3.json().get('accountInfo', {})['daysToLock'] - except Exception: - logging.info(f'Not found licSchet and Expired') - - try: - result['anyString'] = 'Статус: ' + response3.json().get('bonusAccount', {})['tier'] - except Exception: - logging.info(f'Not found anyString (status)') - - session.save_session() - return result +def get_balance(login, password, storename=None): + ''' На вход логин и пароль, на выходе словарь с результатами ''' + return sipnet_over_puppeteer(login, password, storename).main() if __name__ == '__main__': diff --git a/plugin/settings.py b/plugin/settings.py index 1e88e5f..86a7441 100644 --- a/plugin/settings.py +++ b/plugin/settings.py @@ -70,6 +70,8 @@ # 3 - Поступившее через портал городских услуг (ПГУ) 'mosenergosbyt_nm_indication_take': '0', 'mosenergosbyt_nm_indication_variants': '1:ЛКК,2:АИИС КУЭ,3:ПГУ', + # Вести отдельный полный лог по стокам (stock.py) + 'stock_fulllog': '0', # average_days - если нет в Options.ini Additional\AverageDays то возьмем отсюда # Количество дней для расчета среднего по истории 'average_days': 30, diff --git a/plugin/stock.py b/plugin/stock.py index 3843583..cbc7028 100644 --- a/plugin/stock.py +++ b/plugin/stock.py @@ -52,8 +52,8 @@ def get_yahoo(market, security, cnt, qu=None): url = time.strftime(f'https://query1.finance.yahoo.com/v8/finance/chart/{security}') response = session.get(url) meta = response.json()['chart']['result'][0]['meta'] - meta['regularMarketPrice'] - res = meta['regularMarketPrice']*cnt, security, 'USD' + price = meta['regularMarketPrice'] + res = {'security':security, 'price':price,'value':price*cnt, 'cnt': cnt, 'currency':'USD'} if qu: qu.put(res) return res @@ -69,7 +69,8 @@ def get_finex(market, security, cnt, qu=None): "variables": json.dumps({"ticker": security}), "query": "query GetFondDetail($ticker: String!) {fonds(ticker: $ticker){edges {node{price} __typename } __typename }}"} response = session.post(url, data) - res = response.json()['data']['fonds']['edges'][0]['node']['price'], security, 'RUB' # + price = response.json()['data']['fonds']['edges'][0]['node']['price'] + res = {'security':security, 'price':price,'value':price*cnt, 'cnt': cnt, 'currency':'RUB'} if qu: qu.put(res) return res @@ -87,7 +88,8 @@ def get_moex_old(market, security, cnt, qu=None): rows_market = root.findall('*[@id="marketdata"]/rows')[0] prevwarprices = [c.get('PREVWAPRICE') for c in list(rows_securities) if c.get('PREVWAPRICE')!=''] lasts = [c.get('LAST') for c in list(rows_market) if c.get('LAST')!=''] - res = float(lasts[0] if lasts != [] else prevwarprices[0])*cnt, security, 'RUB' + price = float(lasts[0] if lasts != [] else prevwarprices[0]) + res = {'security':security, 'price':price,'value':price*cnt, 'cnt': cnt, 'currency':'RUB'} if qu: qu.put(res) return res @@ -101,7 +103,8 @@ def get_moex(market, security, cnt, qu=None): root=etree.fromstring(response.text) rows = root.find('*[@id="marketdata"]/rows') allsec = {l.items()[0][1]:l.items()[1][1] for l in rows} - res = float(allsec[security.upper()])*cnt, security, marketval[moexmarket] + price = float(allsec[security.upper()]) + res = {'security':security, 'price':price,'value':price*cnt, 'cnt': cnt, 'currency':marketval[moexmarket]} if qu: qu.put(res) return res @@ -141,13 +144,15 @@ def count_all_scocks_multithread(stocks, remain, currenc): while qu.qsize()>0: data.append(qu.get_nowait()) orderlist = list(zip(*stocks))[0] # Порядок, в котором исходно шли бумаги - data.sort(key=lambda i:orderlist.index(i[1])) # Сортируем в исходном порядке - res_full = '\n'.join([f'{sec+"("+curr+")":10} : {round(val*k[curr],2):9.2f} {currenc}' for val,sec,curr in data])+'\n' - res_balance = round(sum([val*k[curr] for val,sec,curr in data]) + remain['USD']*k['USD'] + remain['RUB']*k['RUB'],2) + data.sort(key=lambda i:orderlist.index(i['security'])) # Сортируем в исходном порядке + for line in data: + line['value_priv'] = line['value']*k[line['currency']] + res_full = '\n'.join([f'{i["security"]+"("+i["currency"]+")":10} : {round(i["value_priv"],2):9.2f} {currenc}' for i in data])+'\n' + res_balance = round(sum([i['value_priv'] for i in data]) + remain['USD']*k['USD'] + remain['RUB']*k['RUB'],2) if len(data) != len(stocks): - diff = ','.join(set([i[1] for i in data]).symmetric_difference(set([i[0] for i in stocks]))) + diff = ','.join(set([i['security'] for i in data]).symmetric_difference(set([i[0] for i in stocks]))) raise RuntimeError(f'Not all stock was return ({len(data)} of {len(stocks)}):{diff}') - return res_balance, res_full + return res_balance, res_full, data def get_balance(login, password, storename=None): result = {} @@ -155,7 +160,10 @@ def get_balance(login, password, storename=None): stocks = data['stocks'] remain = data['remain'] currenc = data['currenc'] - res_balance, res_full = count_all_scocks_multithread(stocks, remain, currenc) + res_balance, res_full, res_data = count_all_scocks_multithread(stocks, remain, currenc) + if store.options('stock_fulllog'): + fulllog = '\n'.join(f'{time.strftime("%Y.%m.%d %H:%M:%S",time.localtime())}\t{i["security"]}\t{i["price"]}\t{i["currency"]}\t{i["cnt"]}\t{round(i["value"],2)}\t{round(i["value_priv"],2)}' for i in res_data) + open(os.path.join(store.options('loggingfolder'),f'stock_{login}.log'),'a').write(fulllog+'\n') result['Stock'] = res_full # Полная информация по стокам result['Balance'] = res_balance # Сумма в заданной валюте result['Currenc'] = currenc # Валюта diff --git a/plugin/test3.py b/plugin/test3.py index 1afb1a5..ba601b5 100644 --- a/plugin/test3.py +++ b/plugin/test3.py @@ -1,88 +1,31 @@ #!/usr/bin/python3 # -*- coding: utf8 -*- -import asyncio, time, re, json, subprocess, logging, shutil, os, traceback -import win32gui, win32process, psutil -import pyppeteer # PYthon puPPETEER -#import pprint; pp = pprint.PrettyPrinter(indent=4).pprint -import store, settings import pyppeteeradd as pa -async def async_main(login, password, storename=None): - result = {} - pa.clear_cache(storename) - async def worker(response): - # Если будем смотреть загруженные страницы - то делать это будем здесь - # пока просто демонстрация работы - if response.status != 200: - return - if response.request.url.endswith('/dashboard'): # https://lk.saures.ru/dashboard - logging.info(f'Catch page https://lk.saures.ru/dashboard {len(await response.text())} bytes') - # для страниц json можно брать - # await response.json() - # только оборачивать try except - # или просто текст - # await response.text() - return - - browser, page = await pa.launch_browser(storename, worker) - # Нажмите кнопку "Демо-доступ" или введите логин demo@saures.ru и пароль demo вручную. - await pa.page_goto(page, 'https://lk.saures.ru/dashboard') - - if await pa.page_evaluate(page, "document.getElementById('main-wrapper')!=null"): - logging.info(f'Already login') - else: - for cnt in range(20): # Почему-то иногда с первого раза логон не проскакивает - if await pa.page_evaluate(page, "document.querySelector('form input[type=password]') !== null"): - logging.info(f'Login') - await page.type("#email", login, {'delay': 10}) - await page.type("#password", password, {'delay': 10}) - await asyncio.sleep(1) - # await page.click("form button[type=submit]") # почему-то так не заработало - await pa.page_evaluate(page, "document.querySelector('form button[type=submit]').click()") - elif await pa.page_evaluate(page, "document.getElementById('main-wrapper')!=null"): - logging.info(f'Logoned') - break - await asyncio.sleep(1) - if cnt==10: - await pa.page_reload(page, 'unclear: logged in or not') - else: - logging.error(f'Unknown state') - raise RuntimeError(f'Unknown state') - - # Ждем появления информации - for cnt in range(20): - await asyncio.sleep(1) - if await pa.page_evaluate(page, "document.querySelector('.sensor-5 .d-inline')!=null"): - break - else: - logging.error(f'Not found BALANCE') - raise RuntimeError(f'Not found BALANCE') - - baltext = await pa.page_evaluate(page, "document.querySelector('.sensor-5 .d-inline').innerText") - baltext = re.sub(r'[^\d|,|.-]','',baltext).replace(',', '.') - result['Balance'] = float(baltext) - - block_status = await pa.page_evaluate(page, "document.querySelector('.sensor-9 .d-inline').innerText") - if block_status is not None: - result['BlockStatus'] = block_status - else: - logging.info(f'Not found BlockStatus') - - logging.debug(f'Data ready {result.keys()}') - await browser.close() - pa.clear_cache(storename) - return result +# введите логин demo@saures.ru и пароль demo вручную +class test4_over_puppeteer(pa.balance_over_puppeteer): + async def async_main(self): + await self.do_logon( + url='https://lk.saures.ru/dashboard', + user_selectors={'chk_lk_page_js': "document.querySelector('form input[type=password]') == null", + 'chk_login_page_js': "document.querySelector('form input[type=password]') !== null", + 'login_clear_js': "document.querySelector('form input[type=text]').value=''", + 'login_selector': 'form input[type=text]', }) + # Здесь мы берет данные непосредственно с отрендеренной страницы, поэтому url_tag не указан + await self.wait_params(params=[{ + 'name': 'Balance', + 'jsformula': r"parseFloat(document.querySelector('.sensor-5 .d-inline').innerText.replace(/[^\d,.]/g, '').replace(',','.'))", + }, { + 'name': 'BlockStatus', + 'jsformula': r"document.querySelector('.sensor-9 .d-inline').innerText", + }]) def get_balance(login, password, storename=None): ''' На вход логин и пароль, на выходе словарь с результатами ''' - result = {} - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - result = asyncio.get_event_loop().run_until_complete(async_main(login, password, storename)) - return result + return test4_over_puppeteer(login, password, storename).main() if __name__ == '__main__': - print('This is module test3 for test chrome on puppeteer') + print('This is module test4 for test chrome on puppeteer with class balance_over_puppeteer') diff --git a/plugin/test4.py b/plugin/test4.py index ba601b5..ca942c9 100644 --- a/plugin/test4.py +++ b/plugin/test4.py @@ -1,31 +1,73 @@ #!/usr/bin/python3 # -*- coding: utf8 -*- -import pyppeteeradd as pa - - -# введите логин demo@saures.ru и пароль demo вручную -class test4_over_puppeteer(pa.balance_over_puppeteer): - async def async_main(self): - await self.do_logon( - url='https://lk.saures.ru/dashboard', - user_selectors={'chk_lk_page_js': "document.querySelector('form input[type=password]') == null", - 'chk_login_page_js': "document.querySelector('form input[type=password]') !== null", - 'login_clear_js': "document.querySelector('form input[type=text]').value=''", - 'login_selector': 'form input[type=text]', }) - # Здесь мы берет данные непосредственно с отрендеренной страницы, поэтому url_tag не указан - await self.wait_params(params=[{ - 'name': 'Balance', - 'jsformula': r"parseFloat(document.querySelector('.sensor-5 .d-inline').innerText.replace(/[^\d,.]/g, '').replace(',','.'))", - }, { - 'name': 'BlockStatus', - 'jsformula': r"document.querySelector('.sensor-9 .d-inline').innerText", - }]) +''' пример плагина почти на чистом puppeteer без упрощенной логики ''' +import asyncio, time, re, json, logging, os +import pyppeteer # PYthon puPPETEER +import pyppeteeradd + +async def async_main(login, password, storename=None): + result = {} + pyppeteeradd.clear_cache(storename) + + pa = pyppeteeradd.balance_over_puppeteer(login, password, storename) + await pa.launch_browser() + # Нажмите кнопку "Демо-доступ" или введите логин demo@saures.ru и пароль demo вручную. + await pa.page_goto('https://lk.saures.ru/dashboard') + + if await pa.page_evaluate("document.getElementById('main-wrapper')!=null"): + logging.info(f'Already login') + else: + for cnt in range(20): # Почему-то иногда с первого раза логон не проскакивает + if await pa.page_evaluate("document.querySelector('form input[type=password]') !== null"): + logging.info(f'Login') + await pa.page_type("#email", login, {'delay': 10}) + await pa.page_type("#password", password, {'delay': 10}) + await asyncio.sleep(1) + # await page.click("form button[type=submit]") # почему-то так не заработало + await pa.page_evaluate("document.querySelector('form button[type=submit]').click()") + elif await pa.page_evaluate("document.getElementById('main-wrapper')!=null"): + logging.info(f'Logoned') + break + await asyncio.sleep(1) + if cnt==10: + await pa.page_reload('unclear: logged in or not') + else: + logging.error(f'Unknown state') + raise RuntimeError(f'Unknown state') + + # Ждем появления информации + for cnt in range(20): + await asyncio.sleep(1) + if await pa.page_evaluate("document.querySelector('.sensor-5 .d-inline')!=null"): + break + else: + logging.error(f'Not found BALANCE') + raise RuntimeError(f'Not found BALANCE') + + baltext = await pa.page_evaluate("document.querySelector('.sensor-5 .d-inline').innerText") + baltext = re.sub(r'[^\d|,|.-]','',baltext).replace(',', '.') + result['Balance'] = float(baltext) + + block_status = await pa.page_evaluate("document.querySelector('.sensor-9 .d-inline').innerText") + if block_status is not None: + result['BlockStatus'] = block_status + else: + logging.info(f'Not found BlockStatus') + + logging.debug(f'Data ready {result.keys()}') + await pa.browser.close() + pyppeteeradd.clear_cache(storename) + return result def get_balance(login, password, storename=None): ''' На вход логин и пароль, на выходе словарь с результатами ''' - return test4_over_puppeteer(login, password, storename).main() + result = {} + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = asyncio.get_event_loop().run_until_complete(async_main(login, password, storename)) + return result if __name__ == '__main__': - print('This is module test4 for test chrome on puppeteer with class balance_over_puppeteer') + print('This is module test3 for test chrome on puppeteer')