-
Notifications
You must be signed in to change notification settings - Fork 23
Expand file tree
/
Copy pathfilesystem.py
More file actions
34 lines (28 loc) · 1.04 KB
/
filesystem.py
File metadata and controls
34 lines (28 loc) · 1.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
import logging
from typing import Tuple, Any
from urllib.parse import urlparse
import fsspec
_logger = logging.getLogger(__name__)
def resolve_filesystem_and_path(
uri: str, **kwargs: Any
) -> Tuple[fsspec.AbstractFileSystem, str]:
parsed_uri = urlparse(uri)
fs_path = parsed_uri.path
if parsed_uri.scheme == "hdfs" or parsed_uri.scheme == "viewfs":
netloc_split = parsed_uri.netloc.split(":")
host = netloc_split[0]
if host == "":
host = "default"
else:
host = parsed_uri.scheme + "://" + host
port = 0
if len(netloc_split) == 2 and netloc_split[1].isnumeric():
port = int(netloc_split[1])
fs = fsspec.filesystem("hdfs", host=host, port=port, **kwargs)
elif parsed_uri.scheme == "":
# Input is local path such as /home/user/myfile.parquet
fs = fsspec.filesystem("file", **kwargs)
else:
fs = fsspec.filesystem(parsed_uri.scheme, **kwargs)
_logger.info(f"Resolved base filesystem: {type(fs)}")
return fs, fs_path