From 1314fb4db8258dc17eecd1c5b52b3739f9047680 Mon Sep 17 00:00:00 2001 From: Jared Yu Date: Wed, 29 Jul 2026 14:50:02 -0700 Subject: [PATCH] IO: Fix Windows drive letters misidentified as URI schemes by urlparse On Windows, Python's urlparse treats paths like 'C:\Users\...' as having scheme='c', which causes 'Unrecognized filesystem type in URI: c' errors. This adds a platform-guarded predicate that detects single-character alphabetic schemes on Windows and remaps them to 'file', enabling correct local filesystem routing. Fixes all three parse sites (_infer_file_io_from_scheme, PyArrowFileIO.parse_location, FsspecFileIO._get_fs_from_uri) and includes platform-conditional tests. Related: #2477, #1005 --- pyiceberg/io/__init__.py | 24 +++++++++++++++++++++--- pyiceberg/io/fsspec.py | 3 +++ pyiceberg/io/pyarrow.py | 5 ++++- tests/io/test_io.py | 30 ++++++++++++++++++++++++++++++ tests/io/test_pyarrow.py | 14 ++++++++++++++ 5 files changed, 72 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 7dbc651214..e3e383f05f 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -27,6 +27,7 @@ import importlib import logging +import sys import warnings from abc import ABC, abstractmethod from io import SEEK_SET @@ -41,6 +42,17 @@ logger = logging.getLogger(__name__) + +def _is_windows_drive_letter(scheme: str) -> bool: + r"""Check if a parsed URL scheme is actually a Windows drive letter. + + On Windows, Python's urlparse treats paths like 'C:\\Users\\...' as having + scheme='c'. This detects that case so callers can route to the local filesystem. + Only returns True on Windows — no behavior change on other platforms. + """ + return sys.platform == "win32" and len(scheme) == 1 and scheme.isalpha() + + AWS_PROFILE_NAME = "client.profile-name" AWS_REGION = "client.region" AWS_ACCESS_KEY_ID = "client.access-key-id" @@ -336,13 +348,19 @@ def _import_file_io(io_impl: str, properties: Properties) -> FileIO | None: def _infer_file_io_from_scheme(path: str, properties: Properties) -> FileIO | None: parsed_url = urlparse(path) - if parsed_url.scheme: - if file_ios := SCHEMA_TO_FILE_IO.get(parsed_url.scheme): + scheme = parsed_url.scheme + + if scheme and _is_windows_drive_letter(scheme): + # Windows drive letter (e.g. 'C:\...') misidentified as a URL scheme + scheme = "file" + + if scheme: + if file_ios := SCHEMA_TO_FILE_IO.get(scheme): for file_io_path in file_ios: if file_io := _import_file_io(file_io_path, properties): return file_io else: - warnings.warn(f"No preferred file implementation for scheme: {parsed_url.scheme}", stacklevel=2) + warnings.warn(f"No preferred file implementation for scheme: {scheme}", stacklevel=2) return None diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 7749268ff5..0fd805c2be 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -89,6 +89,7 @@ InputStream, OutputFile, OutputStream, + _is_windows_drive_letter, ) from pyiceberg.typedef import Properties from pyiceberg.types import strtobool @@ -480,6 +481,8 @@ def delete(self, location: str | InputFile | OutputFile) -> None: def _get_fs_from_uri(self, uri: "ParseResult") -> AbstractFileSystem: """Get a filesystem from a parsed URI, using hostname for ADLS account resolution.""" + if _is_windows_drive_letter(uri.scheme): + return self.get_fs("file") if uri.scheme in _ADLS_SCHEMES: return self.get_fs(uri.scheme, uri.hostname) return self.get_fs(uri.scheme) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 29ede3ce9e..4870bbcd9c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -121,6 +121,7 @@ InputStream, OutputFile, OutputStream, + _is_windows_drive_letter, ) from pyiceberg.io.fileformat import DataFileStatistics as DataFileStatistics from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel, FileFormatWriter @@ -404,10 +405,12 @@ def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[ """Return (scheme, netloc, path) for the given location. Uses DEFAULT_SCHEME and DEFAULT_NETLOC if scheme/netloc are missing. + On Windows, single-letter schemes (drive letters like 'C') are treated + as local file paths rather than URI schemes. """ uri = urlparse(location) - if not uri.scheme: + if not uri.scheme or _is_windows_drive_letter(uri.scheme): default_scheme = properties.get("DEFAULT_SCHEME", "file") default_netloc = properties.get("DEFAULT_NETLOC", "") return default_scheme, default_netloc, os.path.abspath(location) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index d9bee33f8b..68fdb18ebd 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -17,6 +17,7 @@ import os import pickle +import sys import tempfile from typing import Any @@ -27,6 +28,7 @@ PY_IO_IMPL, _import_file_io, _infer_file_io_from_scheme, + _is_windows_drive_letter, load_file_io, ) from pyiceberg.io.pyarrow import PyArrowFileIO @@ -339,3 +341,31 @@ def test_infer_file_io_from_schema_unknown() -> None: _infer_file_io_from_scheme("unknown://bucket/path/", {}) assert str(w[0].message) == "No preferred file implementation for scheme: unknown" + + +@pytest.mark.parametrize( + "scheme", + ["s3", "hdfs", "file", "gs", ""], +) +def test_is_windows_drive_letter_false_for_real_schemes(scheme: str) -> None: + assert _is_windows_drive_letter(scheme) is False + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +@pytest.mark.parametrize("scheme", ["c", "D", "z"]) +def test_is_windows_drive_letter_true_on_windows(scheme: str) -> None: + assert _is_windows_drive_letter(scheme) is True + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +def test_infer_file_io_from_scheme_windows_drive_letter() -> None: + # A Windows-style path like 'C:\Users\...' should be treated as a local file path, + # not as a URL with scheme 'c' + result = _infer_file_io_from_scheme(r"C:\Users\test\warehouse", {}) + assert isinstance(result, PyArrowFileIO) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +def test_infer_file_io_from_scheme_windows_drive_letter_lowercase() -> None: + result = _infer_file_io_from_scheme(r"d:\data\iceberg\table", {}) + assert isinstance(result, PyArrowFileIO) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index fef33a25c7..386bea5178 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -17,6 +17,7 @@ # pylint: disable=protected-access,unused-argument,redefined-outer-name import logging import os +import sys import tempfile import uuid import warnings @@ -2326,6 +2327,19 @@ def check_results(location: str, expected_schema: str, expected_netloc: str, exp check_results("/root/tmp/foo.txt", "file", "", "/root/tmp/foo.txt") +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +def test_parse_location_windows_drive_letter() -> None: + """Windows drive letters like 'C:' should be treated as local file paths, not URL schemes.""" + import os + + for drive in ("C", "D", "c", "d"): + path = f"{drive}:\\Users\\test\\file.avro" + scheme, netloc, result_path = PyArrowFileIO.parse_location(path) + assert scheme == "file" + assert netloc == "" + assert result_path == os.path.abspath(path) + + def test_make_compatible_name() -> None: assert make_compatible_name("label/abc") == "label_x2Fabc" assert make_compatible_name("label?abc") == "label_x3Fabc"