Skip to content

Commit

Permalink
feat: installing readr package
Browse files Browse the repository at this point in the history
  • Loading branch information
arthurfg committed Sep 15, 2023
1 parent 5c1a7f2 commit f29ff75
Showing 1 changed file with 16 additions and 1 deletion.
17 changes: 16 additions & 1 deletion pipelines/datasets/br_cvm_fi/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from rpy2.robjects.packages import importr
import rpy2.robjects.packages as rpackages
import rpy2.robjects as ro
from rpy2.robjects.vectors import StrVector
from rpy2.robjects import pandas2ri
from pipelines.datasets.br_cvm_fi.utils import (
sheet_to_df,
Expand Down Expand Up @@ -391,11 +392,25 @@ def clean_data_make_partitions_perfil(diretorio, table_id):
df_final = pd.DataFrame()
arquivos = glob.glob(f"{diretorio}*.csv")

# import R's utility package
utils = rpackages.importr("utils")

# select a mirror for R packages
utils.chooseCRANmirror(ind=1)
# R package names
packnames = "readr"

# R vector of strings
names_to_install = [x for x in packnames if not rpackages.isinstalled(x)]
if len(names_to_install) > 0:
utils.install_packages(StrVector(names_to_install))
# Import readr

readr = rpackages.importr("readr")
for file in tqdm(arquivos):
log(f"Baixando o arquivo ------> {file}")
## reading with R

readr = rpackages.importr("readr")
df_r = readr.read_delim(
file, delim=";", locale=readr.locale(encoding="ISO-8859-1")
)
Expand Down

0 comments on commit f29ff75

Please sign in to comment.