Skip to content

Commit

Permalink
fix(impala): remove no-longer-used temporary database and paths that …
Browse files Browse the repository at this point in the history
…may prevent connection success (#8489)

Remove the creation of temporary resources in the Impala backend, which
are no longer used after the HDFS removal (these temp resources were
used as scratch space for things like dataframe csvs). Fixes #8466.
  • Loading branch information
cpcloud authored Feb 28, 2024
1 parent 1371016 commit 32fcce6
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 29 deletions.
23 changes: 0 additions & 23 deletions ibis/backends/impala/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
wrap_udf,
)
from ibis.backends.sql import SQLBackend
from ibis.config import options

if TYPE_CHECKING:
from collections.abc import Iterator, Mapping
Expand All @@ -68,21 +67,6 @@ class Backend(SQLBackend):

supports_in_memory_tables = True

class Options(ibis.config.Config):
"""Impala specific options.
Parameters
----------
temp_db : str, default "__ibis_tmp"
Database to use for temporary objects.
temp_path : str, default "/tmp/ibis"
Path for storage of temporary data.
"""

temp_db: str = "__ibis_tmp"
temp_path: str = "/tmp/__ibis"

def _from_url(self, url: str, **kwargs: Any) -> Backend:
"""Connect to a backend using a URL `url`.
Expand Down Expand Up @@ -212,7 +196,6 @@ def do_connect(

self.con = con
self.options = {}
self._ensure_temp_db_exists()

@cached_property
def version(self):
Expand Down Expand Up @@ -703,12 +686,6 @@ def _get_concrete_table_path(
) -> tuple[str, str | None]:
return name if name is not None else util.gen_name("impala_table"), database

def _ensure_temp_db_exists(self):
# TODO: session memoize to avoid unnecessary `SHOW DATABASES` calls
name, path = options.impala.temp_db, options.impala.temp_path
if name not in self.list_databases():
self.create_database(name, path=path, force=True)

def _drop_table(self, name: str) -> None:
# database might have been dropped, so we suppress the
# corresponding Exception
Expand Down
8 changes: 2 additions & 6 deletions ibis/backends/impala/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,7 @@ def impala_port(self):
@property
def tmp_dir(self):
leaf = util.gen_name("impala_test_tmp_dir")
options.impala.temp_path = tmp_dir = os.environ.get(
"IBIS_TEST_TMP_DIR", f"/tmp/{leaf}"
)
return tmp_dir
return os.environ.get("IBIS_TEST_TMP_DIR", f"/tmp/{leaf}")

@property
def test_data_db(self):
Expand All @@ -175,8 +172,7 @@ def env():

@pytest.fixture(scope="session")
def tmp_dir(env):
options.impala.temp_path = tmp_dir = env.tmp_dir
return tmp_dir
return env.tmp_dir


@pytest.fixture(scope="session")
Expand Down

0 comments on commit 32fcce6

Please sign in to comment.