Skip to content

Commit

Permalink
test(snowflake): use parquet for data loading in test suite
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud authored and gforsyth committed Aug 22, 2023
1 parent 46d0e33 commit 2be1323
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 12 deletions.
8 changes: 0 additions & 8 deletions ci/schema/snowflake.sql
Original file line number Diff line number Diff line change
@@ -1,11 +1,3 @@
CREATE OR REPLACE TEMP FILE FORMAT ibis_testing
type = 'CSV'
field_delimiter = ','
skip_header = 1
field_optionally_enclosed_by = '"';

CREATE OR REPLACE TEMP STAGE ibis_testing file_format = ibis_testing;

CREATE OR REPLACE TABLE diamonds (
"carat" FLOAT,
"cut" TEXT,
Expand Down
19 changes: 15 additions & 4 deletions ibis/backends/snowflake/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@
import os
from typing import TYPE_CHECKING, Any

import pyarrow.parquet as pq
import pytest
import sqlalchemy as sa

import ibis
from ibis.backends.conftest import TEST_TABLES
from ibis.backends.snowflake.datatypes import SnowflakeType
from ibis.backends.tests.base import BackendTest, RoundAwayFromZero
from ibis.formats.pyarrow import PyArrowSchema

if TYPE_CHECKING:
from pathlib import Path
Expand Down Expand Up @@ -42,12 +45,19 @@ def _get_url():

def copy_into(con, data_dir: Path, table: str) -> None:
stage = "ibis_testing"
csv = f"{table}.csv"
con.exec_driver_sql(
f"PUT file://{data_dir.joinpath('csv', csv).absolute()} @{stage}/{csv}"
file = data_dir.joinpath("parquet", f"{table}.parquet").absolute()
schema = PyArrowSchema.to_ibis(pq.read_metadata(file).schema.to_arrow_schema())
columns = ", ".join(
f"$1:{name}{'::VARCHAR' * typ.is_timestamp()}::{SnowflakeType.to_string(typ)}"
for name, typ in schema.items()
)
con.exec_driver_sql(f"PUT {file.as_uri()} @{stage}/{file.name}")
con.exec_driver_sql(
f"COPY INTO {table} FROM @{stage}/{csv} FILE_FORMAT = (FORMAT_NAME = ibis_testing)"
f"""
COPY INTO {table}
FROM (SELECT {columns} FROM @{stage}/{file.name})
FILE_FORMAT = (TYPE = PARQUET)
"""
)


Expand Down Expand Up @@ -76,6 +86,7 @@ def _load_data(self, **_: Any) -> None:
USE DATABASE ibis_testing;
CREATE SCHEMA IF NOT EXISTS {dbschema};
USE SCHEMA {dbschema};
CREATE TEMP STAGE ibis_testing;
{self.script_dir.joinpath("snowflake.sql").read_text()}"""
)

Expand Down

0 comments on commit 2be1323

Please sign in to comment.