From fe1d2bd275717d34a45753246caf3f92eb961af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Andrade?= Date: Fri, 3 Nov 2023 15:46:39 -0300 Subject: [PATCH] Starting jupyter notebook --- .../Emendas.csv | 0 execucao.py | 90 ----------------- methods/__pycache__/__init__.cpython-311.pyc | Bin 215 -> 0 bytes .../webPageDataScrapers.cpython-311.pyc | Bin 4598 -> 0 bytes methods/extractors/webPageDataScrapers.py | 58 ----------- .../__pycache__/filesSave.cpython-311.pyc | Bin 8090 -> 0 bytes methods/loaders/filesSave.py | 94 ------------------ .../__pycache__/__init__.cpython-311.pyc | Bin 228 -> 0 bytes .../__pycache__/transformData.cpython-311.pyc | Bin 6236 -> 0 bytes methods/transformers/transformData.py | 64 ------------ teste.ipynb | 88 ++++++++++++++++ 11 files changed, 88 insertions(+), 306 deletions(-) rename {Emendas_Parlamentares_202212 => Emenda_Parlamentar_20231103}/Emendas.csv (100%) delete mode 100644 execucao.py delete mode 100644 methods/__pycache__/__init__.cpython-311.pyc delete mode 100644 methods/extractors/__pycache__/webPageDataScrapers.cpython-311.pyc delete mode 100644 methods/extractors/webPageDataScrapers.py delete mode 100644 methods/loaders/__pycache__/filesSave.cpython-311.pyc delete mode 100644 methods/loaders/filesSave.py delete mode 100644 methods/transformers/__pycache__/__init__.cpython-311.pyc delete mode 100644 methods/transformers/__pycache__/transformData.cpython-311.pyc delete mode 100644 methods/transformers/transformData.py create mode 100644 teste.ipynb diff --git a/Emendas_Parlamentares_202212/Emendas.csv b/Emenda_Parlamentar_20231103/Emendas.csv similarity index 100% rename from Emendas_Parlamentares_202212/Emendas.csv rename to Emenda_Parlamentar_20231103/Emendas.csv diff --git a/execucao.py b/execucao.py deleted file mode 100644 index fabf443..0000000 --- a/execucao.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import pandas as pd -import requests as rq -from methods.loaders.filesSave import FileSavers -from methods.transformers.transformData import TransformData -from methods.extractors.webPageDataScrapers import WebPageDataScrapers -from utils.tools import GeneralTools -import utils.logger_config as logger_config -from utils.charts import GeneralCharts -from utils.aws import AboutAWS -import logging -import seaborn as sns - -def main(): - try: - fileSavers = FileSavers() - transformData = TransformData() - webPageDataScrapers = WebPageDataScrapers() - generalTools = GeneralTools() - generalCharts = GeneralCharts() - df = pd.DataFrame() - client = AboutAWS() - # Variável contendo informações das moedas a serem coletadas, aws e banco de dados - jsonData = generalTools.openJson() - logger_config.setup_logger(generalTools.getDate()) - - # Configurando para 4 casas decimais a formatação e exibição dos n°s com ponto flutuante - pd.options.display.float_format = '{:.4f}'.format - logging.info("CONFIGURAÇÃO PARA 4 CASAS DECIMAIS NOS N°S COM PONTO FLUTUANTE REALIZADA COM SUCESSO.") - - file_name = transformData.getFileName(jsonData) - directory_name = f"{jsonData['source']['generalLink']['params']['directory']}{file_name.split('_')[-1].split('.')[0]}" - logging.info("NOME DO DIRETÓRIO COLETADO COM SUCESSO.") - reference_date = file_name.split("_")[-1].split(".")[0] if isinstance(file_name, str) else 'HISTÓRICA' - logging.info("DATA REFERÊNCIA COLETADA COM SUCESSO.") - file_name = iter([file_name]) if isinstance(file_name, str) else iter(file_name) - generalTools.makeDirectory(directory_name) - logging.info("NOME DO ARQUIVO COLETADO COM SUCESSO.") - fund_data = [] - - html, soup = webPageDataScrapers.requestGetDefault(jsonData['source']['generalLink']['url']) - webPageDataScrapers.extractZip(html, "EmendasParlamentares.zip", directory_name) - logging.info("INFORMAÇÕES DO ARQUIVO BAIXADA COM SUCESSO.") - #zip_file = webPageDataScrapers.readZipFile(jsonData['source']['generalLink']['url'], directory_name) - #fund_data.append(fileSavers.readDeepData(zip_file)) - #logging.info("LEITURA E ARMAZENAMENTO DO CONTEÚDO DO ZIP REALIZADO COM SUCESSO.") - - registration_data = fileSavers.readRegistrationData() - logging.info("ARMAZENANDO DADOS DE REGISTRO SOBRE OS CNPJS.") - filtered_fund_data = transformData.gettingMonthlyReturn(fund_data) - final_base = fileSavers.mergeDataFrames(filtered_fund_data, registration_data) - logging.info("MESCLANDO OS DADOS ATRAVÉS DA COLUNA CNPJ E ESCOLHE AS COLUNAS DESEJADAS.") - largest_values, fund_names, quota_values, fund_cnpjs, index, historical_data = fileSavers.gettingTheBestInvestmentFunds(final_base, jsonData['source']['generalLink']['params']['toprange']) - logging.info(f"ORGANIZANDO DADOS E FILTRANDO OS {jsonData['source']['generalLink']['params']['toprange']} MELHORES FUNDOS.") - final_base = transformData.ajustingColumns(final_base) - - # Análise dos 'toprange' (inserido no data.json) fundos com maiores Patrimônios Líquidos - generalCharts.createBarhChart([10, 6], fund_names, fund_cnpjs, largest_values, reference_date, directory_name, jsonData['source']['generalLink']['params']['toprange']) - logging.info("GRÁFICO DE ANÁLISE DOS 'TOPRANGE' FUNDOS COM MAIORES PATRIMÔNIOS LÍQUIDOS.") - - # Análise da Evolução do Valor Patrimonial Líquido dos 'toprange' ao Longo do Tempo - generalCharts.netAssetValueEvolution([10, 8], final_base, 'Evolução do Valor Patrimonial Líquido', 'Datas', 'Valores Patrimoniais Líquidos (R$)', 90, reference_date, directory_name, fund_names, fund_cnpjs, largest_values) - logging.info("GRÁFICO DE ANÁLISE DA EVOLUÇÃO DO VALOR PATRIMONIAL LÍQUIDO DOS 'TOPRANGE' AO LONGO DO TEMPO.") - - # Análise da Distribuição dos Tipos de Fundos ocupados no 'toprange' (TP_FUNDO) - generalCharts.barChart([10, 8], final_base[final_base['DENOM_SOCIAL'].isin(fund_names)], 'DENOM_SOCIAL', 'TP_FUNDO', 'Distribuição dos Tipos de Fundos do TopRange', 'Tipo de Fundo', 'Quantidade por Tipo Fundos (Int)', 45, reference_date, directory_name, 'DistribuicaoTiposMelhoresFundos') - logging.info("GRÁFICO DE ANÁLISE DA DISTRIBUIÇÃO DOS TIPOS DE FUNDOS OCUPADOS NO 'TOPRANGE'.") - - # Análise da Distribuição Geral dos Tipos de Fundos (TP_FUNDO) - generalCharts.barChart([10, 8], final_base, 'DENOM_SOCIAL', 'TP_FUNDO', 'Distribuição Geral dos Tipos de Fundos', 'Tipo de Fundo', 'Quantidade por Tipo Fundos (Int)', 45, reference_date, directory_name, 'DistribuicaoTiposFundos') - logging.info("GRÁFICO DE ANÁLISE DA DISTRIBUIÇÃO GERAL DOS TIPOS DE FUNDOS OCUPADOS NO 'TOPRANGE'.") - - fileSavers.dfToCsv(final_base, f'{directory_name}/FundosInvestimentoGeral_{directory_name.split("_")[-1]}.csv', "\t") - fileSavers.dfToCsv(final_base[final_base['CNPJ_FUNDO'].isin(fund_cnpjs)], f'{directory_name}/FundosInvestimentoTopRange_{directory_name.split("_")[-1]}.csv', "\t") - - s3 = client.createClient('s3') - for arquivos in os.listdir(directory_name): - localfile = f"{directory_name}/{arquivos}" - client.uploadFile(s3, localfile, 'engdadostest', localfile) - logging.info(f"TODOS OS ARQUIVOS (ZIP, CSV E GRÁFICOS) INSERIDOS NA PASTA {directory_name} FORAM INSERIDOS NO BUCKET RESPECTIVO'.") - - except FileNotFoundError as err: - logging.error(f"ERRO: {generalTools.upperCase(err)}, O ARQUIVO JSON (data.json) NÃO FOI ENCONTRADO.") - except (rq.exceptions.HTTPError, rq.exceptions.RequestException) as err: - logging.error(f"ERRO DURANTE A REQUISIÇÃO: {generalTools.upperCase(err)}") - except Exception as err: - logging.error(f"ERRO DESCONHECIDO: {generalTools.upperCase(err)}") - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/methods/__pycache__/__init__.cpython-311.pyc b/methods/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 0758e0df2e4e4d048599bd3025457d7fe74a321d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 215 zcmZ3^%ge<81n;9&Q$h4&5CH>>P{wCAAY(d13PUi1CZpd= z>1kJWS65ecS65e;e~3gv2+F1KS^Pf%p?}j(rTEH?*M}H{?jsecjE)i|%p@4E&n8$8 za|s_{R`<^a5`Lz{!_Nu{q1+B8f*xNe5dzHV;aM>udSgC4GP^CYjX^BJ;v-P`V?{qo zG~uR|=4!X5@gCoHm>Wv%s4zggrP5aYk64fizf$q^2&vXeA2fVWY1iU<4UfPAKfd?tO$2H9eS}`4S~KmBl`l!@en>>ig1{@YnSH z&=1f>CU}(x4ptR_BB()FV90H`hHqR}rg2=cmGLA|(wJDu>JC8@wHOD#*M~f{cFkD5 zZE{Gij;RQQGhgufZZ^`idtsP?rF5q8iqDH zT>=1mZE9Qk-AcE!=LFOzT_v1vI$3Bs=?EuD1elT)4}~@gz|Y%ZE9?h$NNMG=Qr%&N zLUHcBJu=9qRw|cN)p~ZYLPIE3<;XE9s?jU;wyN(9p?P1a}0w%M+|&6^^0;zRo9tgH%s+Sklu&1V$Z-e6v5-G;lPc| zDmQVw;yHu{G(!E(Wi`{~&)l)G_3O`EK4}^@__`Y^JGY|iE^AtDfZ$p43mo-(p5O{_ z7t*EyH?8Uup4-R-Pg&H0;2ahAsE7g&$#ypKj1F*3Re+#_zmo&$DR}(JiLkPX~?ooEJ#!k+psxQ!Hr+^o6ldI84+Su8= z7|V?j>T@nTcO%NVJ~NFC*FQ%z8qkUm|p~3ah(DA+xgjI9S z&`m|XO7zAdqK07~NVcefzzZVVmw)T*7=XGmS;9{yDo4%)qfgZ}vH6eZzdFBsyU_m2 zyx3I`yBx8r$iY|RcRsL%{9rE7XfK|Yz}jKLdO@>$h6awr1`bPI7tT#wx_C&^wVPNP z#mSpyR2rEfkUsPb9gX$I273FC#rpbsrEz5nlA+SHD4Pw@q^WxtUJTpiuiz<6FoCCp zpMjShvZ-u%#-7zarwBU-GiW%*$-y9f|D6ynUeARPo{(o`j8Mv zv!rgxV;}4tjAcLkEMvn|T$wMKNs|zqiDjs5JwC|Ju3!i!miGJykaS$iws`b7wm{?z zPHHL;qoT`_TP`<^ZMT^yG;y0UiEP=5ov~c*hBfH=YZ>3vt7xnR3!pvD-`+fGzuMWvUyjHlso*6^g=o2~A&JA2kgmp^$^S!i;UF zWf1A|1mE%+9P})DbG#R^`Y3LPajIh!+i^UlWOO_Gi$-tbCbljRWle$67f_LH3LYu? z(4KL2t#jXlV?{sLzO9G=EROyxpw_l~haVhx%sVZ;`Ig>7OYdSFg5?Ww@2a>rFYYgh z`yFxrdQ01~eecdncZFp!{`Nmj+n44GO&`<8?;=|Z*Tt5lOU};z#{vMTd2yg14mje# zTKrrgKH>c4`fB{T6TklTC}sOqUOZh8Pdno2f8EduH_ReJx0nm;D@f~@i|m^cf^b_d z>m{|_H0GY#0A-ygo*Z#@o`9McPZq?J&ZbJGtT%l~Cmz`V<=I0|IA`Y&)Vz4OARcx$ zRXUr#8b>G&pyt-)iN&lVd}Q^3y?;OK9~RIvp*1dY&vp%m&K%_aey|-n&)N2oPVRYU zgtmdgbN}G|&>o8hY3@&xby;>pvOH_58J)I8S-zD~^wJ0ssRMDAO-L)W$X*KQ0h3bz z{*1g7*vqjAjqM(wNrv?efQ9N;WEuZn*m4B~82_o25lT4ql=IO0Ai(=~JsvM2K!0oT z(EDHq>xT!l3d9}$oh!*A0<qIpy{{U0{>uj zD@Ly)CUE_XDQ((~)OE=qWen(MQi1;@;nD<~{>LTsjdOz~iZW+;{|rexB?>fcId7SU zE2IF@zoKr?!gg6o;NplpeDR_y0x6T0OZr?M!U=HVlC?)b&c+G66Hb&4@f5iX12h%0 zLIA;O80IC~w%~nhD7fH#HH{A(RIY32fKy-B&_SoZu5E~lEO=j|S)C55t1p{67IS~N zyL5L!Tno3q2urJBDIeZf2=7}E)==|8pn#f}+wSdN5%Z|KfVv&jy@qx!L<(rj-O#&J zYJq+4lJ)>>mRfDQYy&J-Y6q;{QU}m~uj%Q{WaYScTUOLmg>OJlEIZ);$^`)B5wPj# z#{f)Lm9uFT5&YlIht7@-sW5mfA*b?VJ^9qkl$_Jlv2boe&H<}3qF<8zJ))GaZ4+c6 zm=m)ZRTd;!AiY8u21=qT42Vir#7RU&R)x^Tp|L4hyP22Nv20$HaH%xBvWo9~Z2GRr z2|^~9(F9?(xw@5O8;v{QE!749^VAy0FR-tmNSSUaqsS>OuiQ1A>P=bJOjgO?QB7Am zpVwq1j@Ol~5M+firRYG`5WA!Ge8VxD^G+qpDXbT8jnJ+LcTc_n1Vy1pD|AW_Y=e?iKetl znW82uCPH%58 z3|nQtEB9A)B9= z09u=DCYR0!Sc^&Ru!Pcqpc86e^uiPu|(O5Cu)4eQ+Sns4WacNg#8zqfQx=Q=E2xd`u1eE&J4 zG{a_<{Rj>K0LpSeSt9YuTiDu#t((Y-qw2`&MiPcmRc3eB5tg(fPijcpxY+n0^7Qa( zc%iYxbsJo_&UJ4|+Ix%l?$0gF>D*pI+Ek5zQpi}WznTZ|I$4(l?+(UnRnsUzFXY;;<4!V(%t|(nN;(+DoZ|g)JeLEbf}mh4w=yhBW#v< zqY>Gy{u2O8$~}vo2WN_-3!V~p+~AJu-0?Ca5+!cP;D&T=2xWYBarS;;si1S6-+xB* zeoPagp1A)ig$)W=uA|EmIk z*Qr6h72n3%)Ae|7Z0(uo%unGR-hkqf(B=N{_5^n$!855_S^2f}=B2by!JKWi5xm;* z19&C%M*u8&=NI`0qb2Tu!5z@K17&Itm$+*NcTMN6A+-yOh5PR>y{~ioHd7m27(#6w z3U)7~0FJKRxjseBIku)U_$PD`=h>7b=2_i&Yo8NLm%BP>vf?30Y(3xL`T@WC(YeNh z)Cn-(cbNAaqtqRmrba0meg(V+IHM+WXIu%u0?J_kZ)m@Q!-KKyxal%k^m|M$EvlM$ zTgjWQJ4yzeI}`1ilu6a-)TX1xiG=z0REMC=Yzm< zqU7x|yj|-Q<4)4Wn=kp@OR?ok#qmE({&sTJU7Rc(>NXB_m-vvuhjc#ll5bw>*V{)* z{HVc?>ip=s69-zuE%+6g>T3dP+qWx!^UXKgb*;Oordc z(Ez{Kr(Un}k8dt0ml9WArBpm>^uH%)xS!aZk_pDH!Ex#! zrQxlooZhCCLv!pl+Rqf&UsDp?dH}M&aoT4>9hay%_I(zwk5=gpt??T(Qxwd7@H^&q z{-(|@Z9VH9%Cai~SCmM`tSdY?7(45q^&IhoBY!D49QMz=4J-Sjcme2*Mf~9?92tp0 zbo8TeABcn^vFK;N7>vY@%^U;7BUQ|SSUmpOzYfFQ07is@iEuPH=#Ry#cZm40K!dUe z2AQr@K9>?T(~V(^UWHutF3Werijr4|fSFDi%cdJ0)M;e(#x0LrAc8OnVZ2Gp^>;j< zHFss^*^U|$RM&SJ=28C(z&y3iINiha%jTAcXO_o*-}SpAPiLRM^So8>ij=w{Mps1N z*H>!3Y&2ii`ODwD^tG;2w0jgRP8Fh2x}tYnE1m!V&kAkj089Lc!H?)$Jz#fWx;C6> zm`t|;%hM%rx`g$V_#uNI(zkkWnso=x0R`0$Tl(eUqtqWg-JIi3oxwK8r)^#+e)>*Z zx5M$D4kwgT6^RFAlw_hazk-7OK) zu@emjV0ggfW;4@58iOJ@>qbO?$5B19JdN^Vb5m+NF2IPg)xvndz2(#Y{_elM`?T$g zz}e3OXG?)|M&R7)RU`1zQs)Ju^TP9#p7>|oCzgDo;S+UE-0Z4Uj1-=N+Y#JOz&UVB zdZ_ECz!X$MDz%}20*g+L#7vMsI4a6i&AjlBllsIhJp?(CKnw9>_j|KmI9>*=XOrX%7m8cnV)n;|-`Yyv* z^+N#Q{j{_`9Q)|T!y5~pHUBAeKw`v;iP1X0_hnnhV@`kT#LD^SxAjq3A4#u+*GC5# z$TYeev-S}46@gz9xDa{S-ubw_*tg==51xIoTW^n++M`B$bfIsd?@M1hjOe_G<0=rw zHC@6M_a%l%sWJ{PW;!9RJR#eB8+2@miBQO~TA)!LZ8BAn$^yN;vR6Y_lNA}1m|MiK zR1L%&4RbiE^&$(aZ%T!_LwSxZ&{e0qz)m8UNRrqzQ=M(jxh=O1p^ODM=XpcM=i04kcv-!59M6C{Qe72OH6I z7d2VSKnRtQG}9r85CG0)MbHTqs@Z(XIy>2nn#sX15D`@~GC(gVlpdOgMDKP_z zs|vTpEaoPk%TGboQVf$XHw&q}CYpRIH$A!0r=oo$>QT{Bt@aZ%BQGGD+ zxj$jF3@v!Th3|Z)C>R~5zUVmrdB^$HnNmmC=m;_|s_DyoApJrG4UmQ331YOMNlM1L3>L%#bEjq;fFJ_mR z|F$ydi!ei|vA5`93x#Ck^fj!F)7McGn+SAYxAYM{`zUY2ZUZeC#K%KT5KNCCOy#8+ z2sjHSF9<)I5wlj0f^H+kbT!3vWOAC)22~;!#4sucvHA}P(05Z#A~=NrLt)AV047Ur zeG#k=AdnGg2+#mnCIVyW3Q98p7y?)S3d%oU`K>dw(+e)m25`~NG+k7&B~7pV$jVn! zR{o+~w`w-%ayUDeyB;4|r=Wo2AbW0Qq+A0$4(C3s?pyAD+)oNCagx;oK-{~0^6@!r z-@6hbSv>%p1Iy!&v)F!MMI~810G$2H%HsmI?_YJ0tR8q+=dR_!$7Aah6jorrkktbo z--qDf`v4AP^}yqD4%0<&O4lhYt@c>?i)Jg|0yfdkadaoEaF5H0hQ0lBt=0KG7OJ+veMnIG7toPo@AO^f(R%CbJqGg>CA9>dtbtbk$2r85(z*IsS3a5Vgg zso)`Cc9lO~9fMQZ`ko12u&Nb2RuvWyt-AqljZcwpoCVW z>8;F)6Pdg!Yctb=^-g937#nDoEqS)f__y3SOU=$Jzk*fpd{aSrDR8uDdX4hUliwQE ztZzJP)DLz0vqrtG+n+V+oNj;Cs2%gh=2h`DC095kTIPhg&~+hlhJP_LlF~@{~09ttK8WtCbT%U zs5mAhH90@VB|o_|H#M)MIL0+EJvA>Qu_!Z9AthD8B{3x*Su8#!H6AP$AC_2@nw42x zk{IukT9jWI@0(hblbIKjn_7~QpHduCQk0lioR(jd3$h|UJ~J<~BtBlRpz;@oO>TZl fX-=wL5i8IgAeR>N1BnmJjEsyQ7+^#ZGf)fwjuJp9 diff --git a/methods/transformers/__pycache__/transformData.cpython-311.pyc b/methods/transformers/__pycache__/transformData.cpython-311.pyc deleted file mode 100644 index 7437edd43adedf5046ab6a3d76b413d0d6585b71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6236 zcmbUlS!^4}b@t#TQlfc?wxz^Y6wx*vTfQ2kc5Njxb=RUCNw#BJ!Dhu>(v*35ca=yg z?FvB(D20I%fdi?n1MtI0oz?|ZAOZZD{^h3y$!;NH0RsX0E~TPI zhs!tf=6Eyn?%m%tH#ZR|(I5K6H=Kn01Cd&>Y&4#hps`E@B2Xz3HEJqK={Ox_^qNgu zq87>+vqr6YpDk*Gnoil%j;KQ)V^U3NXVgg%nuw0if$AS>dy%LM`kW1Y&Cq9=X{k*D zjGA%RFoFG?0t)aqiW$Q_28m$#hzQotX_yCpqeOQJ)C39Ho*)_tQJQN|%!m@7n#^WX zGT^=|;`Bv6A@=bKKM|AooG8i9J;AF-gXyC|p(8kSCOCQ~I1$^Rr&rbulSu3^m?Fom*52+__XFpiOy$A#jN=GCl7-BV7qE1+v zN6w{^%K7>JbWWL{$nh~zvn50Y?jmZe05<}0va*P-zEIZYg zjm@UTj3Q6Jz#N|aoH$552D zvMJU+=vgLt3V!q!>x;E{raI4Rwn=ZMgpRLoBBT)6V7ETsUK6~~|hR8uae6yD(f2QeP8)K3J@65Fp(!eKzZ_S0HB9VE}N^dM_0aj_V{Bd zk)PeCJ~F|0lP~7zG;!@w-3=83m#L^tnp+@5v34g=qw6Fv*o0$FdI{3Jm02g z@YYzrwckR=|2oktxPMUJ|FPTm=|HLdBs{BqW&df_f4b~Gqq@&LN<6w+b`O;71BS1# z3Q2I;vPnQTAfXHBuOONR0U>8M%FL$VigmrUQScQI^UdOWG4Eg&a*^GlxYaW2|1Cyf zZm|m5-bN8v1>NsP$x}0UC%u@do6pxr-c8@KZ&7!~U+{doOe8i>#b^plVYHFj5unXW`w(Cx(<~6p6QahX#Ef(TJJ=+c0)!n2 z9FDav?selJ!%GRx63fnJ6pf0@X#Jq(P>dMp79a*<2n-t=FT8A4^M>IrST6q!0Hgvf zo=*>{-dESWM=IVU4=%4p%igzC?^|o$;fi3cK3K6dX|*>hO+99}1^qfG}(y;ZljFnu?%!rz@Kdhcc` z?yi!%tJ=DI`N}6(isSdAUooraza06K`%CxNmh$ob((wU!)Zk#@O1X7NZ5=Am1-k0- zuh92wMe1Ht@#MXhiswMdbD+9=Z^41Tb%uBkuG?x`8Ww4vk+T`h2TZX8tw6WQJ-SVr zV@G}D5ST3;HS6w%#$J5Ewbm^Hj&OVE>E;-|mQLRyjK> zLGtU|$c`^0GdDDAj+gkftg-VVFG=YBHG2m99{6KXv!t^b<+|n=0vDZ?lCi9W;jI=q z&p_LpSU^@i0P7;KzA>?=8#;>X`axKjSdgF7gUmW1L(E0ydjibIxfxU+q&m;|O z2C*B=O#_n76|sT6NpW+yn#(47GrN1SPHQ;`+zQyF}BJ^!9jfp%(?(E5f5=nlT3!c!*=W_5~ zQ%mIgpm$TL&^$GZewv1q-VZ8(-fR?gDm6bYDzj3iF|XQb&T*hGzYYNMs=%HlXSMYe z^g{=9u++b9C(V1-T>C1neJitNS5S2YOZroFx0UQ|27j8Sd3Z02C*cKcXYN>ca2`e2 zb9)hJ_*}p|WX%B*P4g^FuFunEHiJKm)&)p&;KDIn3E@&t#2R;mPZMyqtxp)Y;AL#* z&@b2?+Rbw@{VmqLq-@a^975AWr}+*@8C-=cqPhC|BV(i7#8`N^_kt;ZL}GgW7>5RA zKwA_^5staNm}YD;q<&tWm=A5nYHG(wm`rGnX z5G+_YW)7PC%&c4&Q@5w8x>tJ(RxXmiq}C!rB_jM?y!kIe;@~8}(_jU57^eH^?-1)p zqsI!cF=_xYtQL&~SF%MN zX+EdXIYDzOSx%_EDq3#NiqgF1gpUxS0$&zjp@fE2^XMbcHhFY+1G^eikcQ#Yxc)Z) zz*2+9R$J8In`^;vB^U;V^#fD3s&P6>mx5v7*@0`wGJs1>g`Tp#^Koa_{qEJaM?Y0h z43#^F)y`oU_(|EmzuLNgtui{>AgCM`f%^$ovTO!mtse@L^a;Qk(cVXwD)d{`&8sY+AjljxU0!4_1P&EP< zWAQ)muM?;mK@(*eq>2g>8`QUMfL5EOt=M`0$U1?>BOnh|Bk)p|u%+0CG%#3ggFvHw z)%)d3xURhsc!A)9&WFYZjg&WcGdPIM3wiVuTx0cbACSl3mo`IMzF8%ilDkvcL_(Cf zST+++CbVWVYC{ylk!FwcvckVFWi_Xy 0 and VL_PATRIM_LIQ > 0 and VL_QUOTA > 0') - - base_final.reset_index(inplace=True) - base_final = base_final.drop('index', axis=1) - return base_final \ No newline at end of file diff --git a/teste.ipynb b/teste.ipynb new file mode 100644 index 0000000..4abe2bc --- /dev/null +++ b/teste.ipynb @@ -0,0 +1,88 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Importações\n", + "import pandas as pd\n", + "import requests as rq\n", + "import zipfile\n", + "import datetime\n", + "import io\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# Atribuições\n", + "# URL do arquivo zip que você deseja baixar\n", + "\n", + "url = 'https://portaldatransparencia.gov.br/download-de-dados/emendas-parlamentares/UNICO'\n", + "\n", + "headers = {\n", + " 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',\n", + " 'Referer': f\"{url}\"\n", + "}\n", + "\n", + "data = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n", + "\n", + "name_directory = f\"Emenda_Parlamentar_{data.split(' ')[0].replace('-','')}\"\n", + "\n", + "if not os.path.exists(name_directory):\n", + " os.makedirs(name_directory)\n", + "\n", + "response = rq.get(url, headers=headers)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Extraindo arquivos do zip e inserindo no diretório desejado\n", + "with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref:\n", + " #zip_ref.extractall(path=namedirectory)\n", + " zip_ref.extractall(path=name_directory)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# Lendo arquivo CSV usando pd.read_csv\n", + "df_csv = pd.read_csv(f'{name_directory}/Emendas.csv', sep=';', encoding='ISO-8859-1')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}