Skip to content

Commit

Permalink
Cleanups
Browse files Browse the repository at this point in the history
  • Loading branch information
Jay Chia committed Oct 4, 2023
1 parent a921e18 commit 0ab356a
Showing 1 changed file with 0 additions and 27 deletions.
27 changes: 0 additions & 27 deletions daft/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
else:
from typing import Literal

from typing import Any

import pyarrow as pa
from loguru import logger
Expand Down Expand Up @@ -48,32 +47,6 @@ class ListingInfo:
rows: int | None = None


def _get_s3fs_kwargs() -> dict[str, Any]:
"""Get keyword arguments to forward to s3fs during construction"""

try:
import botocore.session
except ImportError:
logger.error(
"Error when importing botocore. Install getdaft[aws] for the required 3rd party dependencies to interact with AWS S3 (https://www.getdaft.io/projects/docs/en/latest/learn/install.html)"
)
raise

kwargs = {}

# If accessing S3 without credentials, use anonymous access: https://github.com/Eventual-Inc/Daft/issues/503
credentials_available = botocore.session.get_session().get_credentials() is not None
if not credentials_available:
logger.warning(
"AWS credentials not found - using anonymous access to S3 which will fail if the bucket you are accessing is not a public bucket. See boto3 documentation for more details on configuring your AWS credentials: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html"
)
kwargs["anon"] = True

# kwargs["default_fill_cache"] = False

return kwargs


def get_protocol_from_path(path: str) -> str:
parsed_scheme = urllib.parse.urlparse(path, allow_fragments=False).scheme
parsed_scheme = parsed_scheme.lower()
Expand Down

0 comments on commit 0ab356a

Please sign in to comment.