From 172c7cef4af04adcbdc6d7f11d35bcf196d7e389 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 20 Jan 2017 14:00:03 -0500 Subject: [PATCH] add snowflake support (#259) --- .gitignore | 1 + CHANGELOG.md | 6 + Dockerfile | 8 +- Makefile | 8 +- circle.yml | 6 +- dbt/adapters/__init__.py | 0 dbt/adapters/cache.py | 5 + dbt/adapters/factory.py | 35 ++ dbt/adapters/postgres.py | 524 ++++++++++++++++++ dbt/adapters/redshift.py | 65 +++ dbt/adapters/snowflake.py | 249 +++++++++ dbt/archival.py | 19 +- dbt/compilation.py | 11 +- dbt/compiled_model.py | 20 +- dbt/contracts/__init__.py | 0 dbt/contracts/connection.py | 48 ++ dbt/exceptions.py | 18 + dbt/flags.py | 1 + dbt/logger.py | 13 +- dbt/main.py | 23 +- dbt/project.py | 32 +- dbt/runner.py | 277 ++++----- dbt/schema.py | 284 +--------- dbt/seeder.py | 18 +- dbt/targets.py | 2 +- dbt/templates.py | 76 +-- docker-compose.yml | 2 + requirements.txt | 4 +- setup.py | 52 +- test.env.sample | 6 + test/integration.sh | 7 - .../integration/001_simple_copy_test/seed.sql | 203 +++---- .../001_simple_copy_test/test_simple_copy.py | 39 +- .../001_simple_copy_test/update.sql | 201 +++---- .../models/dependent_view.sql | 6 - .../002_varchar_widening_test/seed.sql | 210 ++++--- .../test_varchar_widening.py | 32 +- .../002_varchar_widening_test/update.sql | 203 +++---- .../003_simple_reference_test/seed.sql | 211 +++---- .../test_simple_reference.py | 89 ++- .../003_simple_reference_test/update.sql | 201 +++---- .../invalidate_postgres.sql | 12 + .../invalidate_snowflake.sql | 12 + .../004_simple_archive_test/seed.sql | 127 ++--- .../test_simple_archive.py | 30 +- .../004_simple_archive_test/update.sql | 134 ++--- .../005_simple_seed_test/test_simple_seed.py | 3 + .../test_simple_dependency.py | 5 + .../test_simple_dependency_with_configs.py | 6 +- .../007_dry_run_test/test_dry_run.py | 2 + .../test_schema_tests.py | 5 +- .../009_data_tests_test/test_data_tests.py | 4 +- .../010_permission_tests/test_permissions.py | 9 +- .../test_invalid_models.py | 4 + .../test_profile_config.py | 3 + .../test_context_vars.py | 6 +- .../test_pre_post_run_hooks.py | 5 +- .../test_cli_invocation.py | 5 +- .../016_macro_tests/test_macros.py | 4 + .../test_runtime_materialization.py | 5 +- test/integration/base.py | 208 +++++-- test/setup.sh | 2 +- test/setup_db.sh | 3 + test/unit/test_postgres_adapter.py | 47 ++ tox.ini | 71 ++- 65 files changed, 2527 insertions(+), 1400 deletions(-) create mode 100644 dbt/adapters/__init__.py create mode 100644 dbt/adapters/cache.py create mode 100644 dbt/adapters/factory.py create mode 100644 dbt/adapters/postgres.py create mode 100644 dbt/adapters/redshift.py create mode 100644 dbt/adapters/snowflake.py create mode 100644 dbt/contracts/__init__.py create mode 100644 dbt/contracts/connection.py create mode 100644 dbt/exceptions.py create mode 100644 dbt/flags.py create mode 100644 test.env.sample delete mode 100755 test/integration.sh delete mode 100644 test/integration/002_varchar_widening_test/models/dependent_view.sql create mode 100644 test/integration/004_simple_archive_test/invalidate_postgres.sql create mode 100644 test/integration/004_simple_archive_test/invalidate_snowflake.sql create mode 100644 test/unit/test_postgres_adapter.py diff --git a/.gitignore b/.gitignore index 37c536b9cec..37ac9e5bb9f 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ nosetests.xml coverage.xml *,cover .hypothesis/ +test.env # Translations *.mo diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b141c0bdb2..f5329add2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## dbt 0.7.0 (unreleased) + +#### New Features + +- dbt now supports [Snowflake](https://www.snowflake.net/) as a warehouse ([#259](https://github.com/analyst-collective/dbt/pull/259)) + ## dbt 0.6.2 (January 16, 2017) #### Changes diff --git a/Dockerfile b/Dockerfile index f7af20fe7f8..58574511148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python +FROM python:3.5 RUN apt-get update @@ -8,11 +8,7 @@ RUN apt-get install -y python-dev python3-dev RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install virtualenvwrapper - -COPY . /usr/src/app +RUN pip install tox WORKDIR /usr/src/app RUN cd /usr/src/app -RUN ./test/setup.sh - - diff --git a/Makefile b/Makefile index 502b9fd975f..ca3201ece20 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,17 @@ changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ print $$2 }' | grep '\/test_[a-zA-Z_\-\.]\+.py'` -test: test-unit test-integration +test: + @echo "Full test run starting..." + @time docker-compose run test tox test-unit: @echo "Unit test run starting..." - tox -e unit-py27,unit-py35 + @time docker-compose run test tox -e unit-py27,unit-py35,pep8 test-integration: @echo "Integration test run starting..." - @docker-compose run test /usr/src/app/test/integration.sh + @time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py35,integration-snowflake-py27,integration-snowflake-py35 test-new: @echo "Test run starting..." diff --git a/circle.yml b/circle.yml index 26e6086c50b..211b6cbe5ed 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,8 @@ machine: + python: + version: 3.6.0 post: - - pyenv global 2.7.9 3.5.0 + - pyenv global 2.7.12 3.6.0 hosts: database: 127.0.0.1 @@ -13,7 +15,7 @@ dependencies: - pip install --upgrade pip setuptools || true - pip install --upgrade tox tox-pyenv override: - - pyenv local 2.7.9 3.5.0 + - pyenv local 2.7.12 3.6.0 test: override: diff --git a/dbt/adapters/__init__.py b/dbt/adapters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbt/adapters/cache.py b/dbt/adapters/cache.py new file mode 100644 index 00000000000..807bc1a9950 --- /dev/null +++ b/dbt/adapters/cache.py @@ -0,0 +1,5 @@ +import dbt.adapters.postgres as postgres + + +def reset(): + postgres.connection_cache = {} diff --git a/dbt/adapters/factory.py b/dbt/adapters/factory.py new file mode 100644 index 00000000000..d8abb406010 --- /dev/null +++ b/dbt/adapters/factory.py @@ -0,0 +1,35 @@ +import platform + +import dbt.exceptions + +from dbt.adapters.postgres import PostgresAdapter +from dbt.adapters.redshift import RedshiftAdapter + +if platform.system() != 'Windows': + from dbt.adapters.snowflake import SnowflakeAdapter +else: + SnowflakeAdapter = None + + +def get_adapter(profile): + adapter_type = profile.get('type', None) + + if platform.system() == 'Windows' and \ + adapter_type == 'snowflake': + raise dbt.exceptions.NotImplementedException( + "ERROR: 'snowflake' is not supported on Windows.") + + adapters = { + 'postgres': PostgresAdapter, + 'redshift': RedshiftAdapter, + 'snowflake': SnowflakeAdapter, + } + + adapter = adapters.get(adapter_type, None) + + if adapter is None: + raise RuntimeError( + "Invalid adapter type {}!" + .format(adapter_type)) + + return adapter diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py new file mode 100644 index 00000000000..06a74bef52a --- /dev/null +++ b/dbt/adapters/postgres.py @@ -0,0 +1,524 @@ +import copy +import psycopg2 +import re +import time +import yaml + +from contextlib import contextmanager + +import dbt.exceptions +import dbt.flags as flags + +from dbt.contracts.connection import validate_connection +from dbt.logger import GLOBAL_LOGGER as logger +from dbt.schema import Column, READ_PERMISSION_DENIED_ERROR + +connection_cache = {} + +RELATION_PERMISSION_DENIED_MESSAGE = """ +The user '{user}' does not have sufficient permissions to create the model +'{model}' in the schema '{schema}'. Please adjust the permissions of the +'{user}' user on the '{schema}' schema. With a superuser account, execute the +following commands, then re-run dbt. + +grant usage, create on schema "{schema}" to "{user}"; +grant select, insert, delete on all tables in schema "{schema}" to "{user}";""" + +RELATION_NOT_OWNER_MESSAGE = """ +The user '{user}' does not have sufficient permissions to drop the model +'{model}' in the schema '{schema}'. This is likely because the relation was +created by a different user. Either delete the model "{schema}"."{model}" +manually, or adjust the permissions of the '{user}' user in the '{schema}' +schema.""" + + +@contextmanager +def exception_handler(connection, cursor, model_name, query): + handle = connection.get('handle') + schema = connection.get('credentials', {}).get('schema') + + try: + yield + except psycopg2.ProgrammingError as e: + handle.rollback() + error_data = {"model": model_name, + "schema": schema, + "user": connection.get('credentials', {}).get('user')} + if 'must be owner of relation' in e.diag.message_primary: + raise RuntimeError( + RELATION_NOT_OWNER_MESSAGE.format(**error_data)) + elif "permission denied for" in e.diag.message_primary: + raise RuntimeError( + RELATION_PERMISSION_DENIED_MESSAGE.format(**error_data)) + else: + raise e + except Exception as e: + handle.rollback() + logger.debug("Error running SQL: %s", query) + logger.debug("rolling back connection") + raise e + + +class PostgresAdapter: + + # TODO: wrap sql-related things into the adapter rather than having + # the compiler call this to get the context + date_function = 'datenow()' + + @classmethod + def acquire_connection(cls, profile): + # profile requires some marshalling right now because it includes a + # wee bit of global config. + # TODO remove this + credentials = copy.deepcopy(profile) + + credentials.pop('type', None) + credentials.pop('threads', None) + + result = { + 'type': 'postgres', + 'state': 'init', + 'handle': None, + 'credentials': credentials + } + + logger.info('Connecting to postgres.') + + if flags.STRICT_MODE: + validate_connection(result) + + return cls.open_connection(result) + + @staticmethod + def hash_profile(profile): + return ("{}--{}--{}--{}".format( + profile.get('host'), + profile.get('dbname'), + profile.get('schema'), + profile.get('user'), + )) + + @classmethod + def get_connection(cls, profile): + profile_hash = cls.hash_profile(profile) + + if connection_cache.get(profile_hash): + connection = connection_cache.get(profile_hash) + return connection + + connection = cls.acquire_connection(profile) + connection_cache[profile_hash] = connection + + return cls.get_connection(profile) + + @staticmethod + def get_connection_spec(connection): + credentials = connection.get('credentials') + + return ("dbname='{}' user='{}' host='{}' password='{}' port='{}' " + "connect_timeout=10".format( + credentials.get('dbname'), + credentials.get('user'), + credentials.get('host'), + credentials.get('pass'), + credentials.get('port'), + )) + + @classmethod + def open_connection(cls, connection): + if connection.get('state') == 'open': + logger.debug('Connection is already open, skipping open.') + return connection + + result = connection.copy() + + try: + handle = psycopg2.connect(cls.get_connection_spec(connection)) + + result['handle'] = handle + result['state'] = 'open' + except psycopg2.Error as e: + logger.debug("Got an error when attempting to open a postgres " + "connection: '{}'" + .format(e)) + + result['handle'] = None + result['state'] = 'fail' + + raise dbt.exceptions.FailedToConnectException(str(e)) + + return result + + @classmethod + def create_schema(cls, profile, schema, model_name=None): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + query = ('create schema if not exists "{schema}"' + .format(schema=schema)) + + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def dist_qualifier(cls, dist): + return '' + + @classmethod + def sort_qualifier(cls, sort_type, sort): + return '' + + @classmethod + def create_table(cls, profile, schema, table, columns, sort, dist): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + fields = ['"{field}" {data_type}'.format( + field=column.name, data_type=column.data_type + ) for column in columns] + fields_csv = ",\n ".join(fields) + dist = cls.dist_qualifier(dist) + sort = cls.sort_qualifier('compound', sort) + sql = """ + create table if not exists "{schema}"."{table}" ( + {fields} + ) + {dist} {sort} + """.format( + schema=schema, + table=table, + fields=fields_csv, + sort=sort, + dist=dist) + + logger.debug('creating table "%s"."%s"'.format(schema, table)) + + cls.add_query_to_transaction( + sql, connection, table) + + @classmethod + def get_default_schema(cls, profile): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + return connection.get('credentials', {}).get('schema') + + @classmethod + def drop(cls, profile, relation, relation_type, model_name=None): + if relation_type == 'view': + return cls.drop_view(profile, relation, model_name) + elif relation_type == 'table': + return cls.drop_table(profile, relation, model_name) + else: + raise RuntimeError( + "Invalid relation_type '{}'" + .format(relation_type)) + + @classmethod + def drop_view(cls, profile, view, model_name): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + schema = connection.get('credentials', {}).get('schema') + + query = ('drop view if exists "{schema}"."{view}" cascade' + .format( + schema=schema, + view=view)) + + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def drop_table(cls, profile, table, model_name): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + schema = connection.get('credentials', {}).get('schema') + + query = ('drop table if exists "{schema}"."{table}" cascade' + .format( + schema=schema, + table=table)) + + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def truncate(cls, profile, table, model_name=None): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + schema = connection.get('credentials', {}).get('schema') + + query = ('truncate table "{schema}"."{table}"' + .format( + schema=schema, + table=table)) + + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def rename(cls, profile, from_name, to_name, model_name=None): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + schema = connection.get('credentials', {}).get('schema') + + query = ('alter table "{schema}"."{from_name}" rename to "{to_name}"' + .format( + schema=schema, + from_name=from_name, + to_name=to_name)) + + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def execute_model(cls, profile, model): + parts = re.split(r'-- (DBT_OPERATION .*)', model.compiled_contents) + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + for i, part in enumerate(parts): + matches = re.match(r'^DBT_OPERATION ({.*})$', part) + if matches is not None: + instruction_string = matches.groups()[0] + instruction = yaml.safe_load(instruction_string) + function = instruction['function'] + kwargs = instruction['args'] + + def call_expand_target_column_types(kwargs): + kwargs.update({'profile': profile}) + return cls.expand_target_column_types(**kwargs) + + func_map = { + 'expand_column_types_if_needed': + call_expand_target_column_types + } + + func_map[function](kwargs) + else: + handle, cursor = cls.add_query_to_transaction( + part, connection, model.name) + + handle.commit() + + status = cls.get_status(cursor) + cursor.close() + + return status + + @classmethod + def get_missing_columns(cls, profile, + from_schema, from_table, + to_schema, to_table): + """Returns dict of {column:type} for columns in from_table that are + missing from to_table""" + from_columns = {col.name: col for col in + cls.get_columns_in_table( + profile, from_schema, from_table)} + to_columns = {col.name: col for col in + cls.get_columns_in_table( + profile, to_schema, to_table)} + + missing_columns = set(from_columns.keys()) - set(to_columns.keys()) + + return [col for (col_name, col) in from_columns.items() + if col_name in missing_columns] + + @classmethod + def get_columns_in_table(cls, profile, schema_name, table_name): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + query = """ + select column_name, data_type, character_maximum_length + from information_schema.columns + where table_name = '{table_name}' + """.format(table_name=table_name).strip() + + if schema_name is not None: + query += (" AND table_schema = '{schema_name}'" + .format(schema_name=schema_name)) + + handle, cursor = cls.add_query_to_transaction( + query, connection, table_name) + + data = cursor.fetchall() + columns = [] + + for row in data: + name, data_type, char_size = row + column = Column(name, data_type, char_size) + columns.append(column) + + return columns + + @classmethod + def expand_target_column_types(cls, profile, + temp_table, + to_schema, to_table): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + reference_columns = {col.name: col for col in + cls.get_columns_in_table( + profile, None, temp_table)} + target_columns = {col.name: col for col in + cls.get_columns_in_table( + profile, to_schema, to_table)} + + for column_name, reference_column in reference_columns.items(): + target_column = target_columns.get(column_name) + + if target_column is not None and \ + target_column.can_expand_to(reference_column): + new_type = Column.string_type(reference_column.string_size()) + logger.debug("Changing col type from %s to %s in table %s.%s", + target_column.data_type, + new_type, + to_schema, + to_table) + + cls.alter_column_type( + connection, to_schema, to_table, column_name, new_type) + + @classmethod + def alter_column_type(cls, connection, + schema, table, column_name, new_column_type): + """ + 1. Create a new column (w/ temp name and correct type) + 2. Copy data over to it + 3. Drop the existing column (cascade!) + 4. Rename the new column to existing column + """ + + opts = { + "schema": schema, + "table": table, + "old_column": column_name, + "tmp_column": "{}__dbt_alter".format(column_name), + "dtype": new_column_type + } + + query = """ + alter table "{schema}"."{table}" add column "{tmp_column}" {dtype}; + update "{schema}"."{table}" set "{tmp_column}" = "{old_column}"; + alter table "{schema}"."{table}" drop column "{old_column}" cascade; + alter table "{schema}"."{table}" rename column "{tmp_column}" to "{old_column}"; + """.format(**opts).strip() # noqa + + handle, cursor = cls.add_query_to_transaction( + query, connection, table) + + return cls.get_status(cursor) + + @classmethod + def table_exists(cls, profile, schema, table): + tables = cls.query_for_existing(profile, schema) + exists = tables.get(table) is not None + return exists + + @classmethod + def query_for_existing(cls, profile, schema): + query = """ + select tablename as name, 'table' as type from pg_tables + where schemaname = '{schema}' + union all + select viewname as name, 'view' as type from pg_views + where schemaname = '{schema}' + """.format(schema=schema).strip() # noqa + + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + _, cursor = cls.add_query_to_transaction( + query, connection, schema) + results = cursor.fetchall() + + existing = [(name, relation_type) for (name, relation_type) in results] + + return dict(existing) + + @classmethod + def execute_all(cls, profile, queries, model_name=None): + if len(queries) == 0: + return + + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + handle = connection.get('handle') + + for i, query in enumerate(queries): + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + return cls.get_status(cursor) + + @classmethod + def execute_one(cls, profile, query, model_name=None): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + handle = connection.get('handle') + + return cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def commit(cls, profile): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + handle = connection.get('handle') + handle.commit() + + @classmethod + def get_status(cls, cursor): + return cursor.statusmessage + + @classmethod + def add_query_to_transaction(cls, query, connection, model_name=None): + handle = connection.get('handle') + cursor = handle.cursor() + + with exception_handler(connection, cursor, model_name, query): + logger.debug("SQL: %s", query) + pre = time.time() + cursor.execute(query) + post = time.time() + logger.debug( + "SQL status: %s in %0.2f seconds", + cls.get_status(cursor), post-pre) + return handle, cursor diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py new file mode 100644 index 00000000000..63e1ed60f18 --- /dev/null +++ b/dbt/adapters/redshift.py @@ -0,0 +1,65 @@ +import copy + +import dbt.flags as flags + +from dbt.adapters.postgres import PostgresAdapter +from dbt.contracts.connection import validate_connection +from dbt.logger import GLOBAL_LOGGER as logger + + +class RedshiftAdapter(PostgresAdapter): + + date_function = 'getdate()' + + @classmethod + def acquire_connection(cls, profile): + # profile requires some marshalling right now because it includes a + # wee bit of global config. + # TODO remove this + credentials = copy.deepcopy(profile) + + credentials.pop('type', None) + credentials.pop('threads', None) + + result = { + 'type': 'redshift', + 'state': 'init', + 'handle': None, + 'credentials': credentials + } + + logger.info('Connecting to redshift.') + + if flags.STRICT_MODE: + validate_connection(result) + + return cls.open_connection(result) + + @classmethod + def dist_qualifier(cls, dist): + dist_key = dist_key.strip().lower() + + if dist_key in ['all', 'even']: + return 'diststyle({})'.format(dist_key) + else: + return 'diststyle key distkey("{}")'.format(dist_key) + + @classmethod + def sort_qualifier(cls, sort_type, sort): + valid_sort_types = ['compound', 'interleaved'] + if sort_type not in valid_sort_types: + raise RuntimeError( + "Invalid sort_type given: {} -- must be one of {}" + .format(sort_type, valid_sort_types) + ) + + if type(sort_keys) == str: + sort_keys = [sort_keys] + + formatted_sort_keys = ['"{}"'.format(sort_key) + for sort_key in sort_keys] + keys_csv = ', '.join(formatted_sort_keys) + + return "{sort_type} sortkey({keys_csv})".format( + sort_type=sort_type, keys_csv=keys_csv + ) diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake.py new file mode 100644 index 00000000000..ce43dbab965 --- /dev/null +++ b/dbt/adapters/snowflake.py @@ -0,0 +1,249 @@ +from __future__ import absolute_import + +import copy +import re +import time +import yaml + +import snowflake.connector +import snowflake.connector.errors + +from contextlib import contextmanager + +import dbt.exceptions +import dbt.flags as flags + +from dbt.adapters.postgres import PostgresAdapter +from dbt.contracts.connection import validate_connection +from dbt.logger import GLOBAL_LOGGER as logger + +connection_cache = {} + + +@contextmanager +def exception_handler(connection, cursor, model_name, query): + handle = connection.get('handle') + schema = connection.get('credentials', {}).get('schema') + + try: + yield + except snowflake.connector.errors.ProgrammingError as e: + if 'Empty SQL statement' in e.msg: + logger.debug("got empty sql statement, moving on") + else: + handle.rollback() + raise dbt.exceptions.ProgrammingException(str(e)) + except Exception as e: + handle.rollback() + logger.debug("Error running SQL: %s", query) + logger.debug("rolling back connection") + raise e + + +class SnowflakeAdapter(PostgresAdapter): + + date_function = 'CURRENT_TIMESTAMP()' + + @classmethod + def acquire_connection(cls, profile): + + # profile requires some marshalling right now because it includes a + # wee bit of global config. + # TODO remove this + credentials = copy.deepcopy(profile) + + credentials.pop('type', None) + credentials.pop('threads', None) + + result = { + 'type': 'snowflake', + 'state': 'init', + 'handle': None, + 'credentials': credentials + } + + logger.info('Connecting to snowflake.') + + if flags.STRICT_MODE: + validate_connection(result) + + return cls.open_connection(result) + + @staticmethod + def hash_profile(profile): + return ("{}--{}--{}--{}--{}".format( + profile.get('account'), + profile.get('database'), + profile.get('schema'), + profile.get('user'), + profile.get('warehouse'), + )) + + @classmethod + def open_connection(cls, connection): + if connection.get('state') == 'open': + logger.debug('Connection is already open, skipping open.') + return connection + + result = connection.copy() + + try: + credentials = connection.get('credentials', {}) + handle = snowflake.connector.connect( + account=credentials.get('account'), + user=credentials.get('user'), + password=credentials.get('password'), + database=credentials.get('database'), + schema=credentials.get('schema'), + warehouse=credentials.get('warehouse'), + autocommit=False + ) + + result['handle'] = handle + result['state'] = 'open' + except snowflake.connector.errors.Error as e: + logger.debug("Got an error when attempting to open a snowflake " + "connection: '{}'" + .format(e)) + + result['handle'] = None + result['state'] = 'fail' + + raise dbt.exceptions.FailedToConnectException(str(e)) + + return result + + @classmethod + def query_for_existing(cls, profile, schema): + query = """ + select TABLE_NAME as name, TABLE_TYPE as type + from INFORMATION_SCHEMA.TABLES + where TABLE_SCHEMA = '{schema}' + """.format(schema=schema).strip() # noqa + + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + _, cursor = cls.add_query_to_transaction( + query, connection, schema) + results = cursor.fetchall() + + relation_type_lookup = { + 'BASE TABLE': 'table', + 'VIEW': 'view' + } + + existing = [(name, relation_type_lookup.get(relation_type)) + for (name, relation_type) in results] + + return dict(existing) + + @classmethod + def get_status(cls, cursor): + state = cursor.sqlstate + + if state is None: + state = 'SUCCESS' + + return "{} {}".format(state, cursor.rowcount) + + @classmethod + def rename(cls, profile, from_name, to_name, model_name=None): + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + schema = connection.get('credentials', {}).get('schema') + + # in snowflake, if you fail to include the quoted schema in the + # identifier, the new table will have `schema.upper()` as its new + # schema + query = (''' + alter table "{schema}"."{from_name}" + rename to "{schema}"."{to_name}" + '''.format( + schema=schema, + from_name=from_name, + to_name=to_name)).strip() + + handle, cursor = cls.add_query_to_transaction( + query, connection, model_name) + + @classmethod + def execute_model(cls, profile, model): + parts = re.split(r'-- (DBT_OPERATION .*)', model.compiled_contents) + connection = cls.get_connection(profile) + + if flags.STRICT_MODE: + validate_connection(connection) + + # snowflake requires a schema to be specified for temporary tables + # TODO setup templates to be adapter-specific. then we can just use + # the existing schema for temp tables. + cls.add_query_to_transaction( + 'USE SCHEMA "{}"'.format( + connection.get('credentials', {}).get('schema')), + connection) + + for i, part in enumerate(parts): + matches = re.match(r'^DBT_OPERATION ({.*})$', part) + if matches is not None: + instruction_string = matches.groups()[0] + instruction = yaml.safe_load(instruction_string) + function = instruction['function'] + kwargs = instruction['args'] + + def call_expand_target_column_types(kwargs): + kwargs.update({'profile': profile}) + return cls.expand_target_column_types(**kwargs) + + func_map = { + 'expand_column_types_if_needed': + call_expand_target_column_types + } + + func_map[function](kwargs) + else: + handle, cursor = cls.add_query_to_transaction( + part, connection, model.name) + + handle.commit() + + status = cls.get_status(cursor) + cursor.close() + + return status + + @classmethod + def add_query_to_transaction(cls, query, connection, model_name=None): + handle = connection.get('handle') + cursor = handle.cursor() + + # snowflake only allows one query per api call. + queries = query.strip().split(";") + + for individual_query in queries: + # hack -- after the last ';', remove comments and don't run + # empty queries. this avoids using exceptions as flow control, + # and also allows us to return the status of the last cursor + without_comments = re.sub( + re.compile('^.*(--.*)$', re.MULTILINE), + '', individual_query).strip() + + if without_comments == "": + continue + + with exception_handler(connection, cursor, + model_name, individual_query): + logger.debug("SQL: %s", individual_query) + pre = time.time() + cursor.execute(individual_query) + post = time.time() + logger.debug( + "SQL status: %s in %0.2f seconds", + cls.get_status(cursor), post-pre) + + return handle, cursor diff --git a/dbt/archival.py b/dbt/archival.py index 390f0f2e9ee..e29c30b952b 100644 --- a/dbt/archival.py +++ b/dbt/archival.py @@ -1,9 +1,10 @@ from __future__ import print_function -import dbt.targets import dbt.schema import dbt.templates import jinja2 +from dbt.adapters.factory import get_adapter + class Archival(object): @@ -11,9 +12,6 @@ def __init__(self, project, archive_model): self.archive_model = archive_model self.project = project - self.target = dbt.targets.get_target(self.project.run_environment()) - self.schema = dbt.schema.Schema(self.project, self.target) - def compile(self): source_schema = self.archive_model.source_schema target_schema = self.archive_model.target_schema @@ -22,10 +20,13 @@ def compile(self): unique_key = self.archive_model.unique_key updated_at = self.archive_model.updated_at - self.schema.create_schema(target_schema) + profile = self.project.run_environment() + adapter = get_adapter(profile) - source_columns = self.schema.get_columns_in_table( - source_schema, source_table) + adapter.create_schema(profile, target_schema) + + source_columns = adapter.get_columns_in_table( + profile, source_schema, source_table) if len(source_columns) == 0: raise RuntimeError( @@ -40,7 +41,9 @@ def compile(self): ] dest_columns = source_columns + extra_cols - self.schema.create_table( + + adapter.create_table( + profile, target_schema, target_table, dest_columns, diff --git a/dbt/compilation.py b/dbt/compilation.py index db3faff2530..56b5e408bdc 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -13,9 +13,9 @@ from dbt.linker import Linker from dbt.runtime import RuntimeContext -import dbt.targets import dbt.templates +from dbt.adapters.factory import get_adapter from dbt.logger import GLOBAL_LOGGER as logger CompilableEntities = [ @@ -43,7 +43,6 @@ def __init__(self, project, create_template_class, args): self.project.args = args self.macro_generator = None - self.target = self.get_target() def initialize(self): if not os.path.exists(self.project['target-path']): @@ -52,10 +51,6 @@ def initialize(self): if not os.path.exists(self.project['modules-path']): os.makedirs(self.project['modules-path']) - def get_target(self): - target_cfg = self.project.run_environment() - return dbt.targets.get_target(target_cfg) - def model_sources(self, this_project, own_project=None): if own_project is None: own_project = this_project @@ -254,8 +249,8 @@ def get_context(self, linker, model, models, add_dependency=False): context['run_started_at'] = '{{ run_started_at }}' context['invocation_id'] = '{{ invocation_id }}' - # add in context from run target - context.update(self.target.context) + adapter = get_adapter(self.project.run_environment()) + context['sql_now'] = adapter.date_function runtime.update_global(context) diff --git a/dbt/compiled_model.py b/dbt/compiled_model.py index 936dbe0d4c9..39e80bd8658 100644 --- a/dbt/compiled_model.py +++ b/dbt/compiled_model.py @@ -12,7 +12,7 @@ def __init__(self, fqn, data): # these are set just before the models are executed self.tmp_drop_type = None self.final_drop_type = None - self.target = None + self.profile = None self.skip = False self._contents = None @@ -76,12 +76,12 @@ def project(self): @property def schema(self): - if self.target is None: + if self.profile is None: raise RuntimeError( - "`target` not set in compiled model {}".format(self) + "`profile` not set in compiled model {}".format(self) ) else: - return self.target.schema + return get_adapter(self.profile).get_default_schema(self.profile) def should_execute(self, args, existing): if args.non_destructive and \ @@ -98,7 +98,7 @@ def should_rename(self, args): else: return self.materialization in ['table', 'view'] - def prepare(self, existing, target): + def prepare(self, existing, profile): if self.materialization == 'incremental': tmp_drop_type = None final_drop_type = None @@ -108,7 +108,7 @@ def prepare(self, existing, target): self.tmp_drop_type = tmp_drop_type self.final_drop_type = final_drop_type - self.target = target + self.profile = profile def __repr__(self): return "".format( @@ -126,8 +126,8 @@ def should_rename(self): def should_execute(self, args, existing): return True - def prepare(self, existing, target): - self.target = target + def prepare(self, existing, profile): + self.profile = profile def __repr__(self): return "".format( @@ -145,8 +145,8 @@ def should_rename(self): def should_execute(self, args, existing): return True - def prepare(self, existing, target): - self.target = target + def prepare(self, existing, profile): + self.profile = profile def __repr__(self): return "".format( diff --git a/dbt/contracts/__init__.py b/dbt/contracts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbt/contracts/connection.py b/dbt/contracts/connection.py new file mode 100644 index 00000000000..0646c4cd3c2 --- /dev/null +++ b/dbt/contracts/connection.py @@ -0,0 +1,48 @@ +from voluptuous import Schema, Required, All, Any, Extra, Range +from voluptuous.error import MultipleInvalid + +from dbt.exceptions import ValidationException +from dbt.logger import GLOBAL_LOGGER as logger + + +connection_contract = Schema({ + Required('type'): Any('postgres', 'redshift', 'snowflake'), + Required('state'): Any('init', 'open', 'closed', 'fail'), + Required('handle'): Any(None, object), + Required('credentials'): object, +}) + +postgres_credentials_contract = Schema({ + Required('dbname'): str, + Required('host'): str, + Required('user'): str, + Required('pass'): str, + Required('port'): All(int, Range(min=0, max=65535)), + Required('schema'): str, +}) + +snowflake_credentials_contract = Schema({ + Required('account'): str, + Required('user'): str, + Required('password'): str, + Required('database'): str, + Required('schema'): str, + Required('warehouse'): str, +}) + +credentials_mapping = { + 'postgres': postgres_credentials_contract, + 'redshift': postgres_credentials_contract, + 'snowflake': snowflake_credentials_contract, +} + + +def validate_connection(connection): + try: + connection_contract(connection) + + credentials_contract = credentials_mapping.get(connection.get('type')) + credentials_contract(connection.get('credentials')) + except MultipleInvalid as e: + logger.info(e) + raise ValidationException(str(e)) diff --git a/dbt/exceptions.py b/dbt/exceptions.py new file mode 100644 index 00000000000..991bfced3cd --- /dev/null +++ b/dbt/exceptions.py @@ -0,0 +1,18 @@ +class Exception(BaseException): + pass + + +class ValidationException(Exception): + pass + + +class NotImplementedException(Exception): + pass + + +class ProgrammingException(Exception): + pass + + +class FailedToConnectException(Exception): + pass diff --git a/dbt/flags.py b/dbt/flags.py new file mode 100644 index 00000000000..928c20aaeb6 --- /dev/null +++ b/dbt/flags.py @@ -0,0 +1 @@ +STRICT_MODE = False diff --git a/dbt/logger.py b/dbt/logger.py index e564514dbf4..4b5c86bb893 100644 --- a/dbt/logger.py +++ b/dbt/logger.py @@ -2,18 +2,19 @@ import os import sys -# disable logs from other modules, excepting ERROR logs -logging.getLogger('contracts').setLevel(logging.ERROR) -logging.getLogger('requests').setLevel(logging.ERROR) -logging.getLogger('urllib3').setLevel(logging.ERROR) - +# disable logs from other modules, excepting CRITICAL logs +logging.getLogger('botocore').setLevel(logging.CRITICAL) +logging.getLogger('contracts').setLevel(logging.CRITICAL) +logging.getLogger('requests').setLevel(logging.CRITICAL) +logging.getLogger('urllib3').setLevel(logging.CRITICAL) +logging.getLogger('snowflake.connector').setLevel(logging.CRITICAL) # create a global console logger for dbt stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(logging.Formatter('%(message)s')) stdout_handler.setLevel(logging.INFO) -logger = logging.getLogger() +logger = logging.getLogger('dbt') logger.addHandler(stdout_handler) logger.setLevel(logging.DEBUG) diff --git a/dbt/main.py b/dbt/main.py index e1e5f65acf8..87271a81242 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -7,6 +7,7 @@ import re import dbt.version +import dbt.flags as flags import dbt.project as project import dbt.task.run as run_task import dbt.task.compile as compile_task @@ -19,6 +20,7 @@ import dbt.task.archive as archive_task import dbt.tracking import dbt.config as config +import dbt.adapters.cache as adapter_cache def main(args=None): @@ -97,6 +99,11 @@ def run_from_args(parsed): dbt.tracking.track_invocation_end( project=proj, args=parsed, result_type="ok", result=None ) + except dbt.exceptions.NotImplementedException as e: + logger.info('ERROR: {}'.format(e)) + dbt.tracking.track_invocation_end( + project=proj, args=parsed, result_type="error", result=str(e) + ) except Exception as e: dbt.tracking.track_invocation_end( project=proj, args=parsed, result_type="error", result=str(e) @@ -108,6 +115,8 @@ def invoke_dbt(parsed): task = None proj = None + adapter_cache.reset() + try: proj = project.read_project( 'dbt_project.yml', @@ -167,15 +176,14 @@ def invoke_dbt(parsed): def parse_args(args): p = argparse.ArgumentParser( prog='dbt: data build tool', - formatter_class=argparse.RawTextHelpFormatter - ) + formatter_class=argparse.RawTextHelpFormatter) p.add_argument( '--version', action='version', version=dbt.version.get_version_information(), - help="Show version information" - ) + help="Show version information") + p.add_argument( '-d', '--debug', @@ -183,6 +191,13 @@ def parse_args(args): help='''Display debug logging during dbt execution. Useful for debugging and making bug reports.''') + p.add_argument( + '-S', + '--strict', + action='store_true', + help='''Run schema validations at runtime. This will surface + bugs in dbt, but may incur a performance penalty.''') + subs = p.add_subparsers() base_subparser = argparse.ArgumentParser(add_help=False) diff --git a/dbt/project.py b/dbt/project.py index 717d3ed402c..421adcecf6e 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -5,7 +5,11 @@ import sys import hashlib import re +from voluptuous import Schema, Required, Invalid + import dbt.deprecations +import dbt.contracts.connection +from dbt.logger import GLOBAL_LOGGER as logger default_project_cfg = { 'source-paths': ['models'], @@ -120,13 +124,31 @@ def validate(self): raise DbtProjectError( "Project name and version is not provided", self) - required_keys = ['host', 'user', 'pass', 'schema', 'type', - 'dbname', 'port'] - for key in required_keys: - if key not in target_cfg or len(str(target_cfg[key])) == 0: + validator = dbt.contracts.connection.credentials_mapping.get( + target_cfg.get('type'), None) + + if validator is None: + valid_types = ', '.join(validator.keys()) + raise DbtProjectError( + "Expected project configuration '{}' should be one of {}" + .format(key, valid_types), self) + + validator = validator.extend({ + Required('type'): str, + Required('threads'): int, + }) + + try: + validator(target_cfg) + except Invalid as e: + if 'extra keys not allowed' in str(e): + raise DbtProjectError( + "Extra project configuration '{}' is not recognized" + .format('.'.join(e.path)), self) + else: raise DbtProjectError( "Expected project configuration '{}' was not supplied" - .format(key), self) + .format('.'.join(e.path)), self) def hashed_name(self): if self.cfg.get("name", None) is None: diff --git a/dbt/runner.py b/dbt/runner.py index 2353e23f48c..5c1fa904593 100644 --- a/dbt/runner.py +++ b/dbt/runner.py @@ -11,15 +11,17 @@ import yaml from datetime import datetime +from dbt.adapters.factory import get_adapter from dbt.logger import GLOBAL_LOGGER as logger from dbt.compilation import compile_string from dbt.linker import Linker from dbt.templates import BaseCreateTemplate -import dbt.targets from dbt.source import Source from dbt.utils import find_model_by_fqn, find_model_by_name, \ dependency_projects from dbt.compiled_model import make_compiled_model + +import dbt.exceptions import dbt.tracking import dbt.schema @@ -52,15 +54,18 @@ def skipped(self): class BaseRunner(object): - def __init__(self, project, schema_helper): + def __init__(self, project): self.project = project - self.schema_helper = schema_helper + + self.profile = project.run_environment() + self.adapter = get_adapter(self.profile) def pre_run_msg(self, model): raise NotImplementedError("not implemented") def skip_msg(self, model): - return "SKIP relation {}.{}".format(model.target.schema, model.name) + return "SKIP relation {}.{}".format( + self.adapter.get_default_schema(self.profile), model.name) def post_run_msg(self, result): raise NotImplementedError("not implemented") @@ -80,69 +85,13 @@ def pre_run_all(self, models, context): def status(self, result): raise NotImplementedError("not implemented") - def execute_list(self, queries, source): - if len(queries) == 0: - return - - handle = None - - status = 'None' - for i, query in enumerate(queries): - try: - handle, status = self.schema_helper.execute_without_auto_commit(query, handle) # noqa - except psycopg2.ProgrammingError as e: - error_msg = e.diag.message_primary - if error_msg is not None and \ - "permission denied for" in error_msg: - raise RuntimeError( - "Permission denied while running {}".format(source)) - else: - raise - - handle.commit() - return status - - def execute_contents(self, target, model): - parts = re.split(r'-- (DBT_OPERATION .*)', model.compiled_contents) - handle = None - - status = 'None' - for i, part in enumerate(parts): - matches = re.match(r'^DBT_OPERATION ({.*})$', part) - if matches is not None: - instruction_string = matches.groups()[0] - instruction = yaml.safe_load(instruction_string) - function = instruction['function'] - kwargs = instruction['args'] - - func_map = { - 'expand_column_types_if_needed': lambda kwargs: self.schema_helper.expand_column_types_if_needed(**kwargs), # noqa - } - - func_map[function](kwargs) - else: - try: - handle, status = self.schema_helper.execute_without_auto_commit(part, handle) # noqa - except psycopg2.ProgrammingError as e: - if "permission denied for" in e.diag.message_primary: - raise RuntimeError( - dbt.schema.READ_PERMISSION_DENIED_ERROR.format( - model=model.name, - error=str(e).strip(), - user=target.user)) - else: - raise - - handle.commit() - return status - class ModelRunner(BaseRunner): run_type = 'run' def pre_run_msg(self, model): print_vars = { - "schema": model.target.schema, + "schema": self.adapter.get_default_schema(self.profile), "model_name": model.name, "model_type": model.materialization, "info": "START" @@ -155,7 +104,7 @@ def pre_run_msg(self, model): def post_run_msg(self, result): model = result.model print_vars = { - "schema": model.target.schema, + "schema": self.adapter.get_default_schema(self.profile), "model_name": model.name, "model_type": model.materialization, "info": "ERROR creating" if result.errored else "OK created" @@ -175,16 +124,27 @@ def post_run_all_msg(self, results): def status(self, result): return result.status - def execute(self, target, model): + def execute(self, model): + profile = self.project.run_environment() + adapter = get_adapter(profile) + if model.tmp_drop_type is not None: if model.materialization == 'table' and \ self.project.args.non_destructive: - self.schema_helper.truncate(target.schema, model.tmp_name) + adapter.truncate( + profile=profile, + table=model.tmp_name, + model_name=model.name) else: - self.schema_helper.drop( - target.schema, model.tmp_drop_type, model.tmp_name) + adapter.drop( + profile=profile, + relation=model.tmp_name, + relation_type=model.tmp_drop_type, + model_name=model.name) - status = self.execute_contents(target, model) + status = adapter.execute_model( + profile=profile, + model=model) if model.final_drop_type is not None: if model.materialization == 'table' and \ @@ -193,14 +153,21 @@ def execute(self, target, model): # do nothing here pass else: - self.schema_helper.drop( - target.schema, model.final_drop_type, model.name) + adapter.drop( + profile=profile, + relation=model.name, + relation_type=model.final_drop_type, + model_name=model.name) if model.should_rename(self.project.args): - self.schema_helper.rename( - target.schema, - model.tmp_name, - model.name) + adapter.rename( + profile=profile, + from_name=model.tmp_name, + to_name=model.name, + model_name=model.name) + + adapter.commit( + profile=profile) return status @@ -208,15 +175,26 @@ def __run_hooks(self, hooks, context, source): if type(hooks) not in (list, tuple): hooks = [hooks] + target = self.project.get_target() + ctx = { - "target": self.project.get_target(), + "target": target, "state": "start", "invocation_id": context['invocation_id'], "run_started_at": context['run_started_at'] } compiled_hooks = [compile_string(hook, ctx) for hook in hooks] - self.execute_list(compiled_hooks, source) + + profile = self.project.run_environment() + adapter = get_adapter(profile) + + adapter.execute_all( + profile=profile, + queries=compiled_hooks, + model_name=source) + + adapter.commit(profile) def pre_run_all(self, models, context): hooks = self.project.cfg.get('on-run-start', []) @@ -232,13 +210,17 @@ class DryRunner(ModelRunner): def pre_run_msg(self, model): output = ("DRY-RUN model {schema}.{model_name} " - .format(schema=model.target.schema, model_name=model.name)) + .format( + schema=self.adapter.get_default_schema(self.profile), + model_name=model.name)) return output def post_run_msg(self, result): model = result.model output = ("DONE model {schema}.{model_name} " - .format(schema=model.target.schema, model_name=model.name)) + .format( + schema=self.adapter.get_default_schema(self.profile), + model_name=model.name)) return output def pre_run_all_msg(self, models): @@ -249,17 +231,22 @@ def post_run_all_msg(self, results): .format(get_timestamp(), len(results))) def post_run_all(self, models, results, context): + profile = self.project.run_environment() + adapter = get_adapter(profile) + count_dropped = 0 for result in results: if result.errored or result.skipped: continue model = result.model - schema_name = model.target.schema + schema_name = self.adapter.get_default_schema(self.profile) relation_type = ('table' if model.materialization == 'incremental' else 'view') - self.schema_helper.drop(schema_name, relation_type, model.name) + adapter.drop(profile, model.name, relation_type, model.name) count_dropped += 1 + + adapter.commit(profile) logger.info("Dropped {} dry-run models".format(count_dropped)) @@ -323,8 +310,16 @@ def status(self, result): return info - def execute(self, target, model): - rows = self.schema_helper.execute_and_fetch(model.compiled_contents) + def execute(self, model): + profile = self.project.run_environment() + adapter = get_adapter(profile) + + _, cursor = adapter.execute_one( + profile, model.compiled_contents, model.name) + rows = cursor.fetchall() + + cursor.close() + if len(rows) > 1: raise RuntimeError( "Bad test {name}: Returned {num_rows} rows instead of 1" @@ -344,7 +339,7 @@ class ArchiveRunner(BaseRunner): def pre_run_msg(self, model): print_vars = { - "schema": model.target.schema, + "schema": self.adapter.get_default_schema(self.profile), "model_name": model.name, } @@ -355,7 +350,7 @@ def pre_run_msg(self, model): def post_run_msg(self, result): model = result.model print_vars = { - "schema": model.target.schema, + "schema": self.adapter.get_default_schema(self.profile), "model_name": model.name, "info": "ERROR archiving" if result.errored else "OK created" } @@ -373,8 +368,14 @@ def post_run_all_msg(self, results): def status(self, result): return result.status - def execute(self, target, model): - status = self.execute_contents(target, model) + def execute(self, model): + profile = self.project.run_environment() + adapter = get_adapter(profile) + + status = adapter.execute_model( + profile=profile, + model=model) + return status @@ -385,25 +386,36 @@ def __init__(self, project, target_path, graph_type, args): self.graph_type = graph_type self.args = args - self.target = dbt.targets.get_target( - self.project.run_environment(), - self.args.threads) + profile = self.project.run_environment() + + # TODO validate the number of threads + if self.args.threads is None: + self.threads = profile.get('threads', 1) + else: + self.threads = self.args.threads + + adapter = get_adapter(profile) - if self.target.should_open_tunnel(): - logger.info("Opening ssh tunnel to host {}... " - .format(self.target.ssh_host), end="") - sys.stdout.flush() - self.target.open_tunnel_if_needed() - logger.info("Connected") + def call_get_columns_in_table(schema_name, table_name): + return adapter.get_columns_in_table( + profile, schema_name, table_name) - self.schema = dbt.schema.Schema(self.project, self.target) + def call_get_missing_columns(from_schema, from_table, + to_schema, to_table): + return adapter.get_missing_columns( + profile, from_schema, from_table, + to_schema, to_table) + + def call_table_exists(schema, table): + return adapter.table_exists( + profile, schema, table) self.context = { "run_started_at": datetime.now(), "invocation_id": dbt.tracking.invocation_id, - "get_columns_in_table": self.schema.get_columns_in_table, - "get_missing_columns": self.schema.get_missing_columns, - "already_exists": self.schema.table_exists, + "get_columns_in_table": call_get_columns_in_table, + "get_missing_columns": call_get_missing_columns, + "already_exists": call_table_exists, } def deserialize_graph(self): @@ -418,7 +430,7 @@ def deserialize_graph(self): def execute_model(self, runner, model): logger.debug("executing model %s", model) - result = runner.execute(self.target, model) + result = runner.execute(model) return result def safe_execute_model(self, data): @@ -430,6 +442,7 @@ def safe_execute_model(self, data): try: status = self.execute_model(runner, model) except (RuntimeError, + dbt.exceptions.ProgrammingException, psycopg2.ProgrammingError, psycopg2.InternalError) as e: error = "Error executing {filepath}\n{error}".format( @@ -454,8 +467,11 @@ def safe_execute_model(self, data): status=status, execution_time=execution_time) - def as_concurrent_dep_list(self, linker, models, existing, target, + def as_concurrent_dep_list(self, linker, models, existing, limit_to): + profile = self.project.run_environment() + adapter = get_adapter(profile) + model_dependency_list = [] dependency_list = linker.as_dependency_list(limit_to) for node_list in dependency_list: @@ -466,7 +482,7 @@ def as_concurrent_dep_list(self, linker, models, existing, target, except RuntimeError as e: continue if model.should_execute(self.args, existing): - model.prepare(existing, target) + model.prepare(existing, adapter) level.append(model) model_dependency_list.append(level) return model_dependency_list @@ -509,7 +525,7 @@ def execute_models(self, runner, model_dependency_list, on_failure): self.target_path)) return [] - num_threads = self.target.threads + num_threads = self.threads logger.info("Concurrency: {} threads (target='{}')".format( num_threads, self.project.get_target().get('name')) ) @@ -540,7 +556,7 @@ def get_idx(model): models_to_execute = [model for model in model_list if not model.should_skip()] - threads = self.target.threads + threads = self.threads num_models_this_batch = len(models_to_execute) model_index = 0 @@ -625,18 +641,21 @@ def run_from_graph(self, runner, limit_to): context.update(m.context()) m.compile(context) - schema_name = self.target.schema + profile = self.project.run_environment() + adapter = get_adapter(profile) - logger.info("Connecting to redshift") try: - self.schema.create_schema_if_not_exists(schema_name) - except psycopg2.OperationalError as e: - logger.info("ERROR: Could not connect to the target database. Try" - "`dbt debug` for more information") + schema_name = adapter.get_default_schema(profile) + + adapter.create_schema(profile, schema_name) + except (dbt.exceptions.FailedToConnectException, + psycopg2.OperationalError) as e: + logger.info("ERROR: Could not connect to the target database. Try " + "`dbt debug` for more information.") logger.info(str(e)) sys.exit(1) - existing = self.schema.query_for_existing(schema_name) + existing = adapter.query_for_existing(profile, schema_name) if limit_to is None: specified_models = None @@ -649,7 +668,6 @@ def run_from_graph(self, runner, limit_to): linker, relevant_compiled_models, existing, - self.target, specified_models ) @@ -660,35 +678,26 @@ def run_from_graph(self, runner, limit_to): return results - def safe_run_from_graph(self, *args, **kwargs): - try: - return self.run_from_graph(*args, **kwargs) - except: - raise - finally: - if self.target.should_open_tunnel(): - logger.info("Closing SSH tunnel... ", end="") - sys.stdout.flush() - self.target.cleanup() - logger.info("Done") - def run_tests_from_graph(self, test_schemas, test_data): linker = self.deserialize_graph() compiled_models = [make_compiled_model(fqn, linker.get_node(fqn)) for fqn in linker.nodes()] - schema_name = self.target.schema + profile = self.project.run_environment() + adapter = get_adapter(profile) + + schema_name = adapter.get_default_schema(profile) - logger.info("Connecting to redshift") try: - self.schema.create_schema_if_not_exists(schema_name) - except psycopg2.OperationalError as e: + adapter.create_schema(profile, schema_name) + except (dbt.exceptions.FailedToConnectException, + psycopg2.OperationalError) as e: logger.info("ERROR: Could not connect to the target database. Try " "`dbt debug` for more information") logger.info(str(e)) sys.exit(1) - test_runner = TestRunner(self.project, self.schema) + test_runner = TestRunner(self.project) if test_schemas: schema_tests = [m for m in compiled_models @@ -723,13 +732,13 @@ def run_tests(self, test_schemas=False, test_data=False, limit_to=None): return self.run_tests_from_graph(test_schemas, test_data) def run(self, limit_to=None): - runner = ModelRunner(self.project, self.schema) - return self.safe_run_from_graph(runner, limit_to) + runner = ModelRunner(self.project) + return self.run_from_graph(runner, limit_to) def dry_run(self, limit_to=None): - runner = DryRunner(self.project, self.schema) - return self.safe_run_from_graph(runner, limit_to) + runner = DryRunner(self.project) + return self.run_from_graph(runner, limit_to) def run_archive(self): - runner = ArchiveRunner(self.project, self.schema) - return self.safe_run_from_graph(runner, None) + runner = ArchiveRunner(self.project) + return self.run_from_graph(runner, None) diff --git a/dbt/schema.py b/dbt/schema.py index 9544bf9395a..6450f9ef05a 100644 --- a/dbt/schema.py +++ b/dbt/schema.py @@ -56,7 +56,7 @@ def data_type(self): return self.dtype def is_string(self): - return self.dtype in ['text', 'character varying'] + return self.dtype.lower() in ['text', 'character varying'] def string_size(self): if not self.is_string(): @@ -82,285 +82,3 @@ def string_type(cls, size): def __repr__(self): return "".format(self.name, self.data_type) - - -class Schema(object): - def __init__(self, project, target): - self.project = project - self.target = target - - self.schema_cache = {} - self.runtime_existing = self.query_for_existing(self.target.schema) - - def cache_table_columns(self, schema, table, columns): - tid = (schema, table) - - if tid not in self.schema_cache: - self.schema_cache[tid] = columns - - return tid - - def get_table_columns_if_cached(self, schema, table): - tid = (schema, table) - return self.schema_cache.get(tid, None) - - def get_schemas(self): - existing = [] - results = self.execute_and_fetch( - 'select nspname from pg_catalog.pg_namespace') - return [name for (name,) in results] - - def create_schema(self, schema_name): - target_cfg = self.project.run_environment() - user = target_cfg['user'] - - try: - self.execute( - 'create schema if not exists "{}"'.format(schema_name)) - except psycopg2.ProgrammingError as e: - if "permission denied for" in e.diag.message_primary: - raise RuntimeError( - SCHEMA_PERMISSION_DENIED_MESSAGE.format( - schema=schema_name, user=user)) - else: - raise e - - def query_for_existing(self, schema): - sql = """ - select tablename as name, 'table' as type from pg_tables where schemaname = '{schema}' - union all - select viewname as name, 'view' as type from pg_views where schemaname = '{schema}' """.format(schema=schema) # noqa - - results = self.execute_and_fetch(sql) - existing = [(name, relation_type) for (name, relation_type) in results] - - return dict(existing) - - def execute(self, sql): - with self.target.get_handle() as handle: - with handle.cursor() as cursor: - try: - logger.debug("SQL: %s", sql) - pre = time.time() - cursor.execute(sql) - post = time.time() - logger.debug( - "SQL status: %s in %0.2f seconds", - cursor.statusmessage, post-pre) - return cursor.statusmessage - except Exception as e: - self.target.rollback() - logger.debug("Error running SQL: %s", sql) - logger.debug("rolling back connection") - raise e - - def execute_and_fetch(self, sql): - with self.target.get_handle() as handle: - with handle.cursor() as cursor: - try: - logger.debug("SQL: %s", sql) - pre = time.time() - cursor.execute(sql) - post = time.time() - logger.debug( - "SQL status: %s in %0.2f seconds", - cursor.statusmessage, post-pre) - data = cursor.fetchall() - logger.debug("SQL response: %s", data) - return data - except Exception as e: - self.target.rollback() - logger.debug("Error running SQL: %s", sql) - logger.debug("rolling back connection") - raise e - - def execute_and_handle_permissions(self, query, model_name): - try: - return self.execute(query) - except psycopg2.ProgrammingError as e: - error_data = {"model": model_name, - "schema": self.target.schema, - "user": self.target.user} - if 'must be owner of relation' in e.diag.message_primary: - raise RuntimeError( - RELATION_NOT_OWNER_MESSAGE.format(**error_data)) - elif "permission denied for" in e.diag.message_primary: - raise RuntimeError( - RELATION_PERMISSION_DENIED_MESSAGE.format(**error_data)) - else: - raise e - - def execute_without_auto_commit(self, sql, handle=None): - if handle is None: - handle = self.target.get_handle() - - cursor = handle.cursor() - - try: - logger.debug("SQL: %s", sql) - pre = time.time() - cursor.execute(sql) - post = time.time() - logger.debug( - "SQL status: %s in %0.2f seconds", - cursor.statusmessage, post-pre) - return handle, cursor.statusmessage - except Exception as e: - self.target.rollback() - logger.debug("Error running SQL: %s", sql) - logger.debug("rolling back connection") - raise e - finally: - cursor.close() - - def truncate(self, schema, relation): - sql = ('truncate table "{schema}"."{relation}"' - .format(schema=schema, relation=relation)) - logger.debug("dropping table %s.%s", schema, relation) - self.execute_and_handle_permissions(sql, relation) - logger.debug("dropped %s.%s", schema, relation) - - def drop(self, schema, relation_type, relation): - sql = ('drop {relation_type} if exists "{schema}"."{relation}" cascade' - .format( - schema=schema, - relation_type=relation_type, - relation=relation)) - logger.debug("dropping %s %s.%s", relation_type, schema, relation) - self.execute_and_handle_permissions(sql, relation) - logger.debug("dropped %s %s.%s", relation_type, schema, relation) - - def sql_columns_in_table(self, schema_name, table_name): - sql = (""" - select column_name, data_type, character_maximum_length - from information_schema.columns - where table_name = '{table_name}'""" - .format(table_name=table_name).strip()) - - if schema_name is not None: - sql += (" AND table_schema = '{schema_name}'" - .format(schema_name=schema_name)) - - return sql - - def get_columns_in_table(self, schema_name, table_name, use_cached=True): - logger.debug("getting columns in table %s.%s", schema_name, table_name) - - columns = self.get_table_columns_if_cached(schema_name, table_name) - if columns is not None and use_cached: - logger.debug("Found columns (in cache): %s", columns) - return columns - - sql = self.sql_columns_in_table(schema_name, table_name) - results = self.execute_and_fetch(sql) - - columns = [] - for result in results: - column, data_type, char_size = result - col = Column(column, data_type, char_size) - columns.append(col) - - self.cache_table_columns(schema_name, table_name, columns) - - logger.debug("Found columns: %s", columns) - return columns - - def rename(self, schema, from_name, to_name): - rename_query = 'alter table "{schema}"."{from_name}" rename to "{to_name}"'.format(schema=schema, from_name=from_name, to_name=to_name) # noqa - logger.debug( - "renaming model %s.%s --> %s.%s", - schema, from_name, schema, to_name) - self.execute_and_handle_permissions(rename_query, from_name) - logger.debug( - "renamed model %s.%s --> %s.%s", - schema, from_name, schema, to_name) - - def get_missing_columns(self, from_schema, from_table, to_schema, - to_table): - """Returns dict of {column:type} for columns in from_table that are - missing from to_table""" - from_columns = {col.name: col for col in - self.get_columns_in_table(from_schema, from_table)} - to_columns = {col.name: col for col in - self.get_columns_in_table(to_schema, to_table)} - - missing_columns = set(from_columns.keys()) - set(to_columns.keys()) - - return [col for (col_name, col) in from_columns.items() - if col_name in missing_columns] - - def create_table(self, schema, table, columns, sort, dist): - fields = ['"{field}" {data_type}'.format( - field=column.name, data_type=column.data_type - ) for column in columns] - fields_csv = ",\n ".join(fields) - dist = self.target.dist_qualifier(dist) - sort = self.target.sort_qualifier('compound', sort) - sql = 'create table if not exists "{schema}"."{table}" (\n {fields}\n) {dist} {sort};'.format(schema=schema, table=table, fields=fields_csv, sort=sort, dist=dist) # noqa - logger.debug('creating table "%s"."%s"'.format(schema, table)) - self.execute_and_handle_permissions(sql, table) - - def create_schema_if_not_exists(self, schema_name): - schemas = self.get_schemas() - - if schema_name not in schemas: - self.create_schema(schema_name) - - def alter_column_type(self, schema, table, column_name, new_column_type): - """ - 1. Create a new column (w/ temp name and correct type) - 2. Copy data over to it - 3. Drop the existing column (cascade!) - 4. Rename the new column to existing column - """ - - opts = { - "schema": schema, - "table": table, - "old_column": column_name, - "tmp_column": "{}__dbt_alter".format(column_name), - "dtype": new_column_type - } - - sql = """ - alter table "{schema}"."{table}" add column "{tmp_column}" {dtype}; - update "{schema}"."{table}" set "{tmp_column}" = "{old_column}"; - alter table "{schema}"."{table}" drop column "{old_column}" cascade; - alter table "{schema}"."{table}" rename column "{tmp_column}" to "{old_column}"; - """.format(**opts) # noqa - - status = self.execute(sql) - return status - - def expand_column_types_if_needed(self, temp_table, to_schema, to_table): - source_columns = {col.name: col for col in - self.get_columns_in_table(None, temp_table)} - dest_columns = {col.name: col for col in - self.get_columns_in_table(to_schema, to_table)} - - for column_name, source_column in source_columns.items(): - dest_column = dest_columns.get(column_name) - - if dest_column is not None and \ - dest_column.can_expand_to(source_column): - new_type = Column.string_type(source_column.string_size()) - logger.debug("Changing col type from %s to %s in table %s.%s", - dest_column.data_type, - new_type, - to_schema, - to_table) - self.alter_column_type( - to_schema, to_table, column_name, new_type) - - # update these cols in the cache! This is a hack to fix broken - # incremental models for type expansion. TODO - self.cache_table_columns(to_schema, to_table, source_columns) - - def table_exists(self, schema, table): - if schema == self.target.schema: - exists = self.runtime_existing.get(table) is not None - return exists - else: - tables = self.query_for_existing(schema) - exists = tables.get(table) is not None - return exists diff --git a/dbt/seeder.py b/dbt/seeder.py index 768011361ad..3f5d968e306 100644 --- a/dbt/seeder.py +++ b/dbt/seeder.py @@ -1,20 +1,19 @@ - import os import fnmatch from csvkit import table as csv_table, sql as csv_sql from sqlalchemy.dialects import postgresql as postgresql_dialect import psycopg2 -import dbt.targets from dbt.source import Source from dbt.logger import GLOBAL_LOGGER as logger +from dbt.adapters.factory import get_adapter +import dbt.exceptions class Seeder: def __init__(self, project): self.project = project run_environment = self.project.run_environment() - self.target = dbt.targets.get_target(run_environment) def find_csvs(self): return Source(self.project).get_csvs(self.project['data-paths']) @@ -119,8 +118,17 @@ def do_seed(self, schema, cursor, drop_existing): logger.info(str(e)) def seed(self, drop_existing=False): - schema = self.target.schema + profile = self.project.run_environment() + + if profile.get('type') == 'snowflake': + raise dbt.exceptions.NotImplementedException( + "`seed` operation is not supported for snowflake.") + + adapter = get_adapter(profile) + connection = adapter.get_connection(profile) + + schema = connection.get('credentials', {}).get('schema') - with self.target.get_handle() as handle: + with connection.get('handle') as handle: with handle.cursor() as cursor: self.do_seed(schema, cursor, drop_existing) diff --git a/dbt/targets.py b/dbt/targets.py index c6a5613fbdd..2b163bce208 100644 --- a/dbt/targets.py +++ b/dbt/targets.py @@ -143,7 +143,7 @@ def context(self): target_map = { 'postgres': PostgresTarget, - 'redshift': RedshiftTarget + 'redshift': RedshiftTarget, } diff --git a/dbt/templates.py b/dbt/templates.py index ac18f634692..89b992c8f96 100644 --- a/dbt/templates.py +++ b/dbt/templates.py @@ -170,7 +170,7 @@ def wrap(self, opts): SCDArchiveTemplate = u""" - with current_data as ( + with "current_data" as ( select {% raw %} @@ -178,15 +178,15 @@ def wrap(self, opts): "{{ col.name }}" {% if not loop.last %},{% endif %} {% endfor %}, {% endraw %} - {{ updated_at }} as dbt_updated_at, - {{ unique_key }} as dbt_pk, - {{ updated_at }} as valid_from, - null::timestamp as tmp_valid_to + "{{ updated_at }}" as "dbt_updated_at", + "{{ unique_key }}" as "dbt_pk", + "{{ updated_at }}" as "valid_from", + null::timestamp as "tmp_valid_to" from "{{ source_schema }}"."{{ source_table }}" ), - archived_data as ( + "archived_data" as ( select {% raw %} @@ -194,51 +194,53 @@ def wrap(self, opts): "{{ col.name }}" {% if not loop.last %},{% endif %} {% endfor %}, {% endraw %} - {{ updated_at }} as dbt_updated_at, - {{ unique_key }} as dbt_pk, - valid_from, - valid_to as tmp_valid_to + "{{ updated_at }}" as "dbt_updated_at", + "{{ unique_key }}" as "dbt_pk", + "valid_from", + "valid_to" as "tmp_valid_to" from "{{ target_schema }}"."{{ target_table }}" ), - insertions as ( + "insertions" as ( select - current_data.*, - null::timestamp as valid_to - from current_data - left outer join archived_data on archived_data.dbt_pk = current_data.dbt_pk - where archived_data.dbt_pk is null or ( - archived_data.dbt_pk is not null and - current_data.dbt_updated_at > archived_data.dbt_updated_at and - archived_data.tmp_valid_to is null + "current_data".*, + null::timestamp as "valid_to" + from "current_data" + left outer join "archived_data" + on "archived_data"."dbt_pk" = "current_data"."dbt_pk" + where "archived_data"."dbt_pk" is null or ( + "archived_data"."dbt_pk" is not null and + "current_data"."dbt_updated_at" > "archived_data"."dbt_updated_at" and + "archived_data"."tmp_valid_to" is null ) ), - updates as ( + "updates" as ( select - archived_data.*, - current_data.dbt_updated_at as valid_to - from current_data - left outer join archived_data on archived_data.dbt_pk = current_data.dbt_pk - where archived_data.dbt_pk is not null - and archived_data.dbt_updated_at < current_data.dbt_updated_at - and archived_data.tmp_valid_to is null + "archived_data".*, + "current_data"."dbt_updated_at" as "valid_to" + from "current_data" + left outer join "archived_data" + on "archived_data"."dbt_pk" = "current_data"."dbt_pk" + where "archived_data"."dbt_pk" is not null + and "archived_data"."dbt_updated_at" < "current_data"."dbt_updated_at" + and "archived_data"."tmp_valid_to" is null ), - merged as ( + "merged" as ( - select *, 'update' as change_type from updates + select *, 'update' as "change_type" from "updates" union all - select *, 'insert' as change_type from insertions + select *, 'insert' as "change_type" from "insertions" ) select *, - md5(dbt_pk || '|' || dbt_updated_at) as scd_id - from merged + md5("dbt_pk" || '|' || "dbt_updated_at") as "scd_id" + from "merged" """ @@ -281,16 +283,16 @@ class ArchiveInsertTemplate(object): -- DBT_OPERATION {{ function: expand_column_types_if_needed, args: {{ temp_table: "{identifier}__dbt_archival_tmp", to_schema: "{schema}", to_table: "{identifier}"}} }} -update "{schema}"."{identifier}" set valid_to = tmp.valid_to -from "{identifier}__dbt_archival_tmp" as tmp -where tmp.scd_id = "{schema}"."{identifier}".scd_id - and change_type = 'update'; +update "{schema}"."{identifier}" set "valid_to" = "tmp"."valid_to" +from "{identifier}__dbt_archival_tmp" as "tmp" +where "tmp"."scd_id" = "{schema}"."{identifier}"."scd_id" + and "change_type" = 'update'; insert into "{schema}"."{identifier}" ( {dest_cols} ) select {dest_cols} from "{identifier}__dbt_archival_tmp" -where change_type = 'insert'; +where "change_type" = 'insert'; """ def wrap(self, schema, table, query, unique_key): diff --git a/docker-compose.yml b/docker-compose.yml index c59a5ffc102..f8dcb515def 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,8 @@ services: context: . dockerfile: Dockerfile command: "/root/.virtualenvs/dbt/bin/nosetests" + env_file: + - ./test.env volumes: - .:/usr/src/app working_dir: /usr/src/app diff --git a/requirements.txt b/requirements.txt index b729f1091ee..13fcb79d731 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ psycopg2==2.6.1 sqlparse==0.1.19 networkx==1.11 csvkit==0.9.1 -paramiko==2.0.1 -sshtunnel==0.0.8.2 snowplow-tracker==0.7.2 celery==3.1.23 +voluptuous==0.9.3 +snowflake-connector-python==1.3.9 diff --git a/setup.py b/setup.py index af25371ceaa..07ff30594f7 100644 --- a/setup.py +++ b/setup.py @@ -6,30 +6,32 @@ package_version = "0.6.2" setup( - name=package_name, - version=package_version, - description="Data build tool for Analyst Collective", - author="Analyst Collective", - author_email="admin@analystcollective.org", - url="https://github.com/analyst-collective/dbt", - packages=find_packages(), - test_suite='test', - entry_points={ - 'console_scripts': [ - 'dbt = dbt.main:main', + name=package_name, + version=package_version, + description="Data build tool for Analyst Collective", + author="Analyst Collective", + author_email="admin@analystcollective.org", + url="https://github.com/analyst-collective/dbt", + packages=find_packages(), + test_suite='test', + entry_points={ + 'console_scripts': [ + 'dbt = dbt.main:main', + ], + }, + scripts=[ + 'scripts/dbt', + ], + install_requires=[ + 'Jinja2>=2.8', + 'PyYAML>=3.11', + 'psycopg2==2.6.1', + 'sqlparse==0.1.19', + 'networkx==1.11', + 'csvkit==0.9.1', + 'snowplow-tracker==0.7.2', + 'celery==3.1.23', + 'voluptuous==0.9.3', + 'snowflake-connector-python==1.3.8', ], - }, - scripts=[ - 'scripts/dbt', - ], - install_requires=[ - 'Jinja2>=2.8', - 'PyYAML>=3.11', - 'psycopg2==2.6.1', - 'sqlparse==0.1.19', - 'networkx==1.11', - 'csvkit==0.9.1', - 'snowplow-tracker==0.7.2', - 'celery==3.1.23', - ], ) diff --git a/test.env.sample b/test.env.sample new file mode 100644 index 00000000000..7804276a550 --- /dev/null +++ b/test.env.sample @@ -0,0 +1,6 @@ +DBT_INVOCATION_ENV=development +SNOWFLAKE_TEST_ACCOUNT= +SNOWFLAKE_TEST_USER= +SNOWFLAKE_TEST_PASSWORD= +SNOWFLAKE_TEST_DATABASE= +SNOWFLAKE_TEST_WAREHOUSE= diff --git a/test/integration.sh b/test/integration.sh deleted file mode 100755 index a6a6876af56..00000000000 --- a/test/integration.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -. /usr/src/app/test/setup.sh -workon dbt - -cd /usr/src/app -tox -e integration-py27,integration-py35 diff --git a/test/integration/001_simple_copy_test/seed.sql b/test/integration/001_simple_copy_test/seed.sql index e5ae2c462c1..b4f1b9e9b41 100644 --- a/test/integration/001_simple_copy_test/seed.sql +++ b/test/integration/001_simple_copy_test/seed.sql @@ -1,4 +1,4 @@ -create table simple_copy_001.seed ( +create table "simple_copy_001"."seed" ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -8,103 +8,104 @@ create table simple_copy_001.seed ( ); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); +insert into "simple_copy_001"."seed" (first_name, last_name, email, gender, ip_address) values +('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), +('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), +('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), +('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), +('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), +('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), +('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), +('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), +('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), +('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), +('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), +('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), +('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), +('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), +('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), +('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), +('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), +('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), +('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), +('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), +('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), +('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), +('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), +('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), +('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), +('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), +('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), +('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), +('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), +('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), +('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), +('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), +('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), +('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), +('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), +('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), +('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), +('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), +('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), +('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), +('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), +('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), +('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), +('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), +('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), +('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), +('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), +('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), +('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), +('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), +('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), +('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), +('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), +('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), +('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), +('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), +('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), +('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), +('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), +('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), +('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), +('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), +('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), +('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), +('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), +('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), +('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), +('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), +('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), +('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), +('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), +('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), +('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), +('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), +('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), +('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), +('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), +('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), +('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), +('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), +('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), +('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), +('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), +('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), +('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), +('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), +('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), +('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), +('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), +('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), +('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), +('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), +('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), +('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), +('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), +('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), +('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), +('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), +('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), +('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index f5481cd1801..a56a8a95499 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -1,11 +1,10 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestSimpleCopy(DBTIntegrationTest): def setUp(self): - DBTIntegrationTest.setUp(self) - - self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + pass @property def schema(self): @@ -15,7 +14,12 @@ def schema(self): def models(self): return "test/integration/001_simple_copy_test/models" - def test_simple_copy(self): + @attr(type='postgres') + def test__postgres__simple_copy(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + self.run_dbt() self.assertTablesEqual("seed","view") @@ -30,10 +34,35 @@ def test_simple_copy(self): self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - def test_dbt_doesnt_run_empty_models(self): + @attr(type='postgres') + def test__postgres__dbt_doesnt_run_empty_models(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + self.run_dbt() models = self.get_models_in_schema() self.assertFalse('empty' in models.keys()) self.assertFalse('disabled' in models.keys()) + + @attr(type='snowflake') + def test__snowflake__simple_copy(self): + self.use_default_project() + self.use_profile('snowflake') + self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + + self.run_dbt() + + self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","incremental") + self.assertTablesEqual("seed","materialized") + + self.run_sql_file("test/integration/001_simple_copy_test/update.sql") + + self.run_dbt() + + self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","incremental") + self.assertTablesEqual("seed","materialized") diff --git a/test/integration/001_simple_copy_test/update.sql b/test/integration/001_simple_copy_test/update.sql index 182757f1e66..36eb85c7d5e 100644 --- a/test/integration/001_simple_copy_test/update.sql +++ b/test/integration/001_simple_copy_test/update.sql @@ -1,100 +1,101 @@ -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117'); +insert into "simple_copy_001"."seed" (first_name, last_name, email, gender, ip_address) values +('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), +('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), +('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), +('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'), +('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'), +('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'), +('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'), +('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'), +('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'), +('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'), +('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'), +('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'), +('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'), +('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'), +('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'), +('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'), +('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'), +('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'), +('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'), +('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'), +('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'), +('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'), +('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'), +('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'), +('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'), +('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'), +('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'), +('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'), +('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'), +('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'), +('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'), +('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'), +('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'), +('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'), +('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'), +('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'), +('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'), +('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'), +('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'), +('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'), +('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'), +('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'), +('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'), +('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'), +('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'), +('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'), +('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'), +('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'), +('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'), +('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'), +('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'), +('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'), +('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'), +('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'), +('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'), +('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'), +('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'), +('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'), +('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'), +('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'), +('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'), +('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'), +('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'), +('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'), +('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'), +('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'), +('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'), +('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'), +('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'), +('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'), +('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'), +('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'), +('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'), +('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'), +('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'), +('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'), +('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'), +('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'), +('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'), +('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'), +('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'), +('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'), +('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'), +('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'), +('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'), +('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'), +('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'), +('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'), +('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'), +('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'), +('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'), +('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'), +('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'), +('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'), +('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'), +('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'), +('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'), +('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'), +('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'), +('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117'); diff --git a/test/integration/002_varchar_widening_test/models/dependent_view.sql b/test/integration/002_varchar_widening_test/models/dependent_view.sql deleted file mode 100644 index 2ad213ac487..00000000000 --- a/test/integration/002_varchar_widening_test/models/dependent_view.sql +++ /dev/null @@ -1,6 +0,0 @@ - -{{ - config(materialized='view') -}} - -select gender from {{ ref('materialized') }} diff --git a/test/integration/002_varchar_widening_test/seed.sql b/test/integration/002_varchar_widening_test/seed.sql index dcd959c20f0..d2be8219a73 100644 --- a/test/integration/002_varchar_widening_test/seed.sql +++ b/test/integration/002_varchar_widening_test/seed.sql @@ -1,4 +1,4 @@ -create table varchar_widening_002.seed ( +create table "varchar_widening_002"."seed" ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -7,110 +7,104 @@ create table varchar_widening_002.seed ( ip_address VARCHAR(20) ); - -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); - - --- make sure that column type altering works with dependent views -create view varchar_widening_002.dependent_view_expected as ( - select gender from varchar_widening_002.seed -); +insert into "varchar_widening_002"."seed" (first_name, last_name, email, gender, ip_address) values +('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), +('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), +('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), +('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), +('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), +('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), +('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), +('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), +('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), +('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), +('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), +('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), +('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), +('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), +('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), +('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), +('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), +('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), +('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), +('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), +('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), +('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), +('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), +('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), +('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), +('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), +('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), +('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), +('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), +('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), +('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), +('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), +('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), +('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), +('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), +('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), +('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), +('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), +('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), +('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), +('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), +('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), +('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), +('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), +('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), +('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), +('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), +('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), +('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), +('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), +('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), +('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), +('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), +('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), +('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), +('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), +('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), +('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), +('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), +('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), +('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), +('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), +('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), +('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), +('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), +('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), +('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), +('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), +('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), +('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), +('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), +('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), +('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), +('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), +('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), +('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), +('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), +('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), +('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), +('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), +('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), +('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), +('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), +('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), +('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), +('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), +('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), +('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), +('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), +('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), +('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), +('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), +('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), +('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), +('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), +('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), +('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), +('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), +('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), +('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); diff --git a/test/integration/002_varchar_widening_test/test_varchar_widening.py b/test/integration/002_varchar_widening_test/test_varchar_widening.py index 533a862a9ba..fcf9a6b7d83 100644 --- a/test/integration/002_varchar_widening_test/test_varchar_widening.py +++ b/test/integration/002_varchar_widening_test/test_varchar_widening.py @@ -1,11 +1,10 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestVarcharWidening(DBTIntegrationTest): def setUp(self): - DBTIntegrationTest.setUp(self) - - self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") + pass @property def schema(self): @@ -15,12 +14,34 @@ def schema(self): def models(self): return "test/integration/002_varchar_widening_test/models" - def test_varchar_widening(self): + @attr(type='postgres') + def test__postgres__varchar_widening(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") + + self.run_dbt() + + self.assertTablesEqual("seed","incremental") + self.assertTablesEqual("seed","materialized") + + self.run_sql_file("test/integration/002_varchar_widening_test/update.sql") + + self.run_dbt() + + self.assertTablesEqual("seed","incremental") + self.assertTablesEqual("seed","materialized") + + @attr(type='snowflake') + def test__snowflake__varchar_widening(self): + self.use_default_project() + self.use_profile('snowflake') + self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") + self.run_dbt() self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - self.assertTablesEqual("dependent_view_expected","dependent_view") self.run_sql_file("test/integration/002_varchar_widening_test/update.sql") @@ -28,4 +49,3 @@ def test_varchar_widening(self): self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - self.assertTablesEqual("dependent_view_expected","dependent_view") diff --git a/test/integration/002_varchar_widening_test/update.sql b/test/integration/002_varchar_widening_test/update.sql index aa171152f99..cd52d0ccf7f 100644 --- a/test/integration/002_varchar_widening_test/update.sql +++ b/test/integration/002_varchar_widening_test/update.sql @@ -1,102 +1,103 @@ -ALTER TABLE varchar_widening_002.seed ALTER COLUMN gender TYPE varchar(300); +ALTER TABLE "varchar_widening_002"."seed" ALTER COLUMN gender TYPE varchar(300); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Annie', 'Reynolds', 'areynolds0@nifty.com', 'Amerisource Bergen', '133.30.242.211'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Doris', 'Wood', 'dwood1@skyrock.com', 'Bliss World, LLC', '128.229.89.207'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Ray', 'aray2@google.co.jp', 'Nelco Laboratories, Inc.', '109.74.153.45'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Frank', 'Morgan', 'fmorgan3@1688.com', 'ALK-Abello, Inc.', '252.211.209.9'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Stanley', 'astanley4@google.fr', 'Gemini Pharmaceuticals, Inc. dba ONDRA Pharmaceuticals', '134.142.194.184'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Jordan', 'rjordan5@nymag.com', 'Watson Pharma, Inc.', '195.104.60.172'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kathleen', 'Ryan', 'kryan6@scientificamerican.com', 'SHISEIDO AMERICAS CORPORATION', '209.110.160.192'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Margaret', 'Jacobs', 'mjacobs7@example.com', 'Cardinal Health', '72.36.52.20'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ernest', 'Brown', 'ebrown8@360.cn', 'West-ward Pharmaceutical Corp', '138.157.61.255'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Phillips', 'ephillips9@japanpost.jp', 'Cellex-C International Inc', '68.46.195.188'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Annie', 'Ellis', 'aellisa@weather.com', 'NATURE REPUBLIC CO., LTD.', '163.128.214.142'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Melissa', 'Olson', 'molsonb@theguardian.com', 'Nelco Laboratories, Inc.', '202.22.153.188'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Timothy', 'Martinez', 'tmartinezc@zimbio.com', 'Lake Erie Medical & Surgical Supply DBA Quality Care Products LLC', '45.64.205.47'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Mark', 'Nelson', 'mnelsond@bloomberg.com', '7-Eleven', '91.99.195.160'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kenneth', 'Hart', 'kharte@berkeley.edu', 'Preferred Pharmaceuticals, Inc.', '207.240.9.102'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'White', 'kwhitef@csmonitor.com', 'Cantrell Drug Company', '191.178.162.18'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Mary', 'Greene', 'mgreeneg@usnews.com', 'Neutrogena Corporation', '251.226.65.64'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Bruce', 'Peters', 'bpetersh@blogspot.com', 'Sun & Skin Care Research, LLC', '153.227.91.121'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Albert', 'Armstrong', 'aarmstrongi@weather.com', 'Access Business Group LLC', '199.146.159.228'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Beverly', 'Gray', 'bgrayj@spiegel.de', 'Church & Dwight Co., Inc.', '47.3.135.226'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Catherine', 'Taylor', 'ctaylork@walmart.com', 'Matrixx Initiatives, Inc.', '82.24.129.147'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Paula', 'Bradley', 'pbradleyl@edublogs.org', 'Nash-Finch Company', '14.145.193.163'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Terry', 'Campbell', 'tcampbellm@artisteer.com', 'MedVantx, Inc.', '89.181.95.177'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Bruce', 'Stevens', 'bstevensn@ucla.edu', 'Global Pharmaceuticals', '128.81.126.144'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Bishop', 'rbishopo@telegraph.co.uk', 'General Injectables & Vaccines, Inc.', '191.191.17.173'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Denise', 'Duncan', 'dduncanp@reference.com', 'Bare Escentuals Beauty, Inc.', '150.207.3.163'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Perkins', 'dperkinsq@1und1.de', 'Altaire Pharmaceuticals Inc.', '21.150.103.133'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Brandon', 'Ray', 'brayr@psu.edu', 'Meijer Distribution Inc', '216.53.187.191'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ernest', 'Graham', 'egrahams@tinyurl.com', 'BioComp Pharma, Inc.', '49.85.236.162'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Denise', 'Matthews', 'dmatthewst@digg.com', 'Procter & Gamble Manufacturing Co.', '160.4.119.137'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Randy', 'Alexander', 'ralexanderu@goo.gl', 'Reckitt Benckiser Pharmaceuticals Inc', '211.72.176.12'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Jackson', 'ajacksonv@gizmodo.com', 'Molton Brown LTD (UK)', '226.178.48.73'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Turner', 'wturnerw@reverbnation.com', 'American Health Packaging', '43.22.122.56'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Stephen', 'Ferguson', 'sfergusonx@kickstarter.com', 'Amneal Pharmaceuticals of New York, LLC', '110.211.112.233'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jane', 'Bradley', 'jbradleyy@usgs.gov', 'Kroger Company', '186.153.255.125'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Phillip', 'Wood', 'pwoodz@about.com', 'Unit Dose Services', '112.65.6.93'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jeffrey', 'Howell', 'jhowell10@symantec.com', 'Midlothian Laboratories', '232.92.208.248'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Harvey', 'hharvey11@nhs.uk', 'Novartis Pharmaceuticals Corporation', '50.212.26.218'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Johnston', 'bjohnston12@diigo.com', 'Nelco Laboratories, Inc.', '131.109.13.9'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ernest', 'Burke', 'eburke13@toplist.cz', 'Apotex Corp.', '151.176.178.175'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Joe', 'Wright', 'jwright14@mapy.cz', 'MULTALER & CIE S.A.', '233.55.33.63'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ronald', 'Griffin', 'rgriffin15@topsy.com', 'Gavis Pharmaceuticals, LLC', '174.233.67.86'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Susan', 'Oliver', 'soliver16@goo.gl', 'Bath & Body Works, Inc.', '104.171.43.12'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Karen', 'Cox', 'kcox17@hp.com', 'Home Sweet Homeopathics', '225.51.182.192'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Antonio', 'Larson', 'alarson18@gov.uk', 'Eight and Company', '243.118.98.188'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Brandon', 'Cook', 'bcook19@mozilla.com', 'Chain Drug Consortium, LLC', '38.64.44.255'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gary', 'Gray', 'ggray1a@alexa.com', 'Lil'' Drug Store Products, Inc', '43.34.161.60'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Doris', 'Harrison', 'dharrison1b@wiley.com', 'Dispensing Solutions, Inc.', '153.66.74.140'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Clarence', 'Perry', 'cperry1c@issuu.com', 'Nelco Laboratories, Inc.', '14.72.110.59'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Emily', 'George', 'egeorge1d@blogtalkradio.com', 'State of Florida DOH Central Pharmacy', '148.35.114.224'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Larson', 'dlarson1e@trellian.com', 'G&W Laboratories, Inc.', '134.158.117.11'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Ashley', 'Peters', 'apeters1f@de.vu', 'Mylan Pharmaceuticals Inc.', '50.193.252.146'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Douglas', 'Andrews', 'dandrews1g@mac.com', 'Jubilant HollisterStier LLC', '159.134.237.86'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Dunn', 'cdunn1h@cornell.edu', 'Antigen Laboratories, Inc.', '227.11.100.112'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Heather', 'Black', 'hblack1i@harvard.edu', 'Hospira, Inc.', '61.9.121.22'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'Ruiz', 'sruiz1j@tmall.com', 'Hankuk Bowonbio Co., Ltd', '171.144.250.254'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Carl', 'Martinez', 'cmartinez1k@geocities.jp', 'ALK-Abello, Inc.', '128.216.69.116'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Stephen', 'Anderson', 'sanderson1l@odnoklassniki.ru', 'Cardinal Health', '145.154.63.186'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Diana', 'Payne', 'dpayne1m@ftc.gov', 'Pharmaceutical Associates, Inc.', '98.9.155.136'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gonzalez', 'jgonzalez1n@walmart.com', 'SHISEIDO CO., LTD.', '73.96.109.149'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Cole', 'scole1o@flickr.com', 'Walgreen Company', '251.244.20.117'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Ellis', 'jellis1p@time.com', 'Jubilant HollisterStier LLC', '188.153.76.182'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Hamilton', 'ahamilton1q@dailymail.co.uk', 'ALK-Abello, Inc.', '229.58.149.141'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Kennedy', 'skennedy1r@nifty.com', 'Newton Laboratories, Inc.', '227.105.251.134'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Sara', 'Grant', 'sgrant1s@flickr.com', 'Rubbermaid Commercial Products LLC', '96.211.162.73'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Joan', 'Bennett', 'jbennett1t@forbes.com', 'Nelco Laboratories, Inc.', '143.27.240.163'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Judith', 'Daniels', 'jdaniels1u@theguardian.com', 'Newton Laboratories, Inc.', '164.99.249.153'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Irene', 'Bennett', 'ibennett1v@comsenz.com', 'Cellab Co., Ltd.', '112.104.12.122'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Perez', 'kperez1w@phpbb.com', 'Temple Industrial Welding Supply Co', '211.31.214.131'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Kim', 'jkim1x@umich.edu', 'Bryant Ranch Prepack', '245.252.150.110'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Walter', 'Hernandez', 'whernandez1y@nbcnews.com', 'Virtus Pharmaceuticals LLC', '200.201.83.21'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Larry', 'Scott', 'lscott1z@quantcast.com', 'BIOKEY INC.', '122.141.109.98'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Palmer', 'gpalmer20@usgs.gov', 'JAFRA COSMETICS INTERNATIONAL', '60.173.159.145'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Harry', 'Andrews', 'handrews21@alexa.com', 'NCS HealthCare of KY, Inc dba Vangard Labs', '210.64.37.91'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Jerry', 'Morrison', 'jmorrison22@drupal.org', 'Teva Pharmaceuticals USA Inc', '83.190.174.61'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Irene', 'Diaz', 'idiaz23@joomla.org', 'Dolgencorp, LLC', '214.16.44.235'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Brenda', 'Hansen', 'bhansen24@wisc.edu', 'REMEDYREPACK INC.', '167.231.200.232'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Williamson', 'cwilliamson25@w3.org', 'Kroger Company', '251.202.210.204'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('David', 'Fuller', 'dfuller26@canalblog.com', 'Supervalu Inc', '175.125.205.131'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Norma', 'Bishop', 'nbishop27@jugem.jp', 'Mylan Institutional Inc.', '208.162.25.149'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Brenda', 'Daniels', 'bdaniels28@mediafire.com', 'Space Brands Limited', '92.235.250.138'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kathy', 'Reed', 'kreed29@prweb.com', 'Rugby Laboratories Inc.', '182.114.174.63'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Long', 'along2a@dropbox.com', 'Fresenius Kabi USA, LLC', '160.146.121.173'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Palmer', 'cpalmer2b@desdev.cn', 'Bio-Pharm, Inc.', '135.77.134.24'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Rachel', 'Banks', 'rbanks2c@devhub.com', 'Sam''s West Inc', '35.72.5.193'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kenneth', 'Peters', 'kpeters2d@ocn.ne.jp', 'International Labs, Inc.', '11.38.191.65'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Susan', 'Clark', 'sclark2e@ed.gov', 'Shionogi Inc.', '19.243.67.80'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Walter', 'Sullivan', 'wsullivan2f@vinaora.com', 'STAT Rx USA LLC', '154.137.170.227'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Kathleen', 'Wood', 'kwood2g@salon.com', 'Freds Inc', '155.54.131.149'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Phyllis', 'Henderson', 'phenderson2h@walmart.com', 'REMEDYREPACK INC.', '146.65.150.251'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Cheryl', 'Wells', 'cwells2i@gov.uk', 'Rebel Distributors Corp', '69.127.148.31'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Bradley', 'rbradley2j@un.org', 'Hi-Tech Pharmacal Co., Inc.', '150.101.165.102'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Moreno', 'amoreno2k@tinypic.com', 'Pharmacia and Upjohn Company', '50.27.226.40'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Campbell', 'acampbell2l@auda.org.au', 'Chi Research, Inc.', '242.64.63.241'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Rebecca', 'Butler', 'rbutler2m@godaddy.com', 'Cardinal Health', '40.55.159.66'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Justin', 'Rodriguez', 'jrodriguez2n@meetup.com', 'Hikma Pharmaceutical', '118.9.132.156'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Nelson', 'dnelson2o@narod.ru', 'Nature''s Way Products, Inc.', '165.174.28.134'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Edward', 'Lawson', 'elawson2p@addtoany.com', 'Apotheca Company', '135.17.238.170'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Paul', 'Bell', 'pbell2q@simplemachines.org', 'Washington Homeopathic Products', '235.149.137.62'); -insert into varchar_widening_002.seed (first_name, last_name, email, gender, ip_address) values ('Mark', 'Rose', 'mrose2r@google.pl', 'AMERICAN SALES COMPANY', '164.108.170.187'); +insert into "varchar_widening_002"."seed" (first_name, last_name, email, gender, ip_address) values +('Annie', 'Reynolds', 'areynolds0@nifty.com', 'Amerisource Bergen', '133.30.242.211'), +('Doris', 'Wood', 'dwood1@skyrock.com', 'Bliss World, LLC', '128.229.89.207'), +('Andrea', 'Ray', 'aray2@google.co.jp', 'Nelco Laboratories, Inc.', '109.74.153.45'), +('Frank', 'Morgan', 'fmorgan3@1688.com', 'ALK-Abello, Inc.', '252.211.209.9'), +('Angela', 'Stanley', 'astanley4@google.fr', 'Gemini Pharmaceuticals, Inc. dba ONDRA Pharmaceuticals', '134.142.194.184'), +('Ruby', 'Jordan', 'rjordan5@nymag.com', 'Watson Pharma, Inc.', '195.104.60.172'), +('Kathleen', 'Ryan', 'kryan6@scientificamerican.com', 'SHISEIDO AMERICAS CORPORATION', '209.110.160.192'), +('Margaret', 'Jacobs', 'mjacobs7@example.com', 'Cardinal Health', '72.36.52.20'), +('Ernest', 'Brown', 'ebrown8@360.cn', 'West-ward Pharmaceutical Corp', '138.157.61.255'), +('Elizabeth', 'Phillips', 'ephillips9@japanpost.jp', 'Cellex-C International Inc', '68.46.195.188'), +('Annie', 'Ellis', 'aellisa@weather.com', 'NATURE REPUBLIC CO., LTD.', '163.128.214.142'), +('Melissa', 'Olson', 'molsonb@theguardian.com', 'Nelco Laboratories, Inc.', '202.22.153.188'), +('Timothy', 'Martinez', 'tmartinezc@zimbio.com', 'Lake Erie Medical & Surgical Supply DBA Quality Care Products LLC', '45.64.205.47'), +('Mark', 'Nelson', 'mnelsond@bloomberg.com', '7-Eleven', '91.99.195.160'), +('Kenneth', 'Hart', 'kharte@berkeley.edu', 'Preferred Pharmaceuticals, Inc.', '207.240.9.102'), +('Kathryn', 'White', 'kwhitef@csmonitor.com', 'Cantrell Drug Company', '191.178.162.18'), +('Mary', 'Greene', 'mgreeneg@usnews.com', 'Neutrogena Corporation', '251.226.65.64'), +('Bruce', 'Peters', 'bpetersh@blogspot.com', 'Sun & Skin Care Research, LLC', '153.227.91.121'), +('Albert', 'Armstrong', 'aarmstrongi@weather.com', 'Access Business Group LLC', '199.146.159.228'), +('Beverly', 'Gray', 'bgrayj@spiegel.de', 'Church & Dwight Co., Inc.', '47.3.135.226'), +('Catherine', 'Taylor', 'ctaylork@walmart.com', 'Matrixx Initiatives, Inc.', '82.24.129.147'), +('Paula', 'Bradley', 'pbradleyl@edublogs.org', 'Nash-Finch Company', '14.145.193.163'), +('Terry', 'Campbell', 'tcampbellm@artisteer.com', 'MedVantx, Inc.', '89.181.95.177'), +('Bruce', 'Stevens', 'bstevensn@ucla.edu', 'Global Pharmaceuticals', '128.81.126.144'), +('Ruby', 'Bishop', 'rbishopo@telegraph.co.uk', 'General Injectables & Vaccines, Inc.', '191.191.17.173'), +('Denise', 'Duncan', 'dduncanp@reference.com', 'Bare Escentuals Beauty, Inc.', '150.207.3.163'), +('Dennis', 'Perkins', 'dperkinsq@1und1.de', 'Altaire Pharmaceuticals Inc.', '21.150.103.133'), +('Brandon', 'Ray', 'brayr@psu.edu', 'Meijer Distribution Inc', '216.53.187.191'), +('Ernest', 'Graham', 'egrahams@tinyurl.com', 'BioComp Pharma, Inc.', '49.85.236.162'), +('Denise', 'Matthews', 'dmatthewst@digg.com', 'Procter & Gamble Manufacturing Co.', '160.4.119.137'), +('Randy', 'Alexander', 'ralexanderu@goo.gl', 'Reckitt Benckiser Pharmaceuticals Inc', '211.72.176.12'), +('Aaron', 'Jackson', 'ajacksonv@gizmodo.com', 'Molton Brown LTD (UK)', '226.178.48.73'), +('Wanda', 'Turner', 'wturnerw@reverbnation.com', 'American Health Packaging', '43.22.122.56'), +('Stephen', 'Ferguson', 'sfergusonx@kickstarter.com', 'Amneal Pharmaceuticals of New York, LLC', '110.211.112.233'), +('Jane', 'Bradley', 'jbradleyy@usgs.gov', 'Kroger Company', '186.153.255.125'), +('Phillip', 'Wood', 'pwoodz@about.com', 'Unit Dose Services', '112.65.6.93'), +('Jeffrey', 'Howell', 'jhowell10@symantec.com', 'Midlothian Laboratories', '232.92.208.248'), +('Howard', 'Harvey', 'hharvey11@nhs.uk', 'Novartis Pharmaceuticals Corporation', '50.212.26.218'), +('Benjamin', 'Johnston', 'bjohnston12@diigo.com', 'Nelco Laboratories, Inc.', '131.109.13.9'), +('Ernest', 'Burke', 'eburke13@toplist.cz', 'Apotex Corp.', '151.176.178.175'), +('Joe', 'Wright', 'jwright14@mapy.cz', 'MULTALER & CIE S.A.', '233.55.33.63'), +('Ronald', 'Griffin', 'rgriffin15@topsy.com', 'Gavis Pharmaceuticals, LLC', '174.233.67.86'), +('Susan', 'Oliver', 'soliver16@goo.gl', 'Bath & Body Works, Inc.', '104.171.43.12'), +('Karen', 'Cox', 'kcox17@hp.com', 'Home Sweet Homeopathics', '225.51.182.192'), +('Antonio', 'Larson', 'alarson18@gov.uk', 'Eight and Company', '243.118.98.188'), +('Brandon', 'Cook', 'bcook19@mozilla.com', 'Chain Drug Consortium, LLC', '38.64.44.255'), +('Gary', 'Gray', 'ggray1a@alexa.com', 'Lil'' Drug Store Products, Inc', '43.34.161.60'), +('Doris', 'Harrison', 'dharrison1b@wiley.com', 'Dispensing Solutions, Inc.', '153.66.74.140'), +('Clarence', 'Perry', 'cperry1c@issuu.com', 'Nelco Laboratories, Inc.', '14.72.110.59'), +('Emily', 'George', 'egeorge1d@blogtalkradio.com', 'State of Florida DOH Central Pharmacy', '148.35.114.224'), +('Dennis', 'Larson', 'dlarson1e@trellian.com', 'G&W Laboratories, Inc.', '134.158.117.11'), +('Ashley', 'Peters', 'apeters1f@de.vu', 'Mylan Pharmaceuticals Inc.', '50.193.252.146'), +('Douglas', 'Andrews', 'dandrews1g@mac.com', 'Jubilant HollisterStier LLC', '159.134.237.86'), +('Craig', 'Dunn', 'cdunn1h@cornell.edu', 'Antigen Laboratories, Inc.', '227.11.100.112'), +('Heather', 'Black', 'hblack1i@harvard.edu', 'Hospira, Inc.', '61.9.121.22'), +('Shirley', 'Ruiz', 'sruiz1j@tmall.com', 'Hankuk Bowonbio Co., Ltd', '171.144.250.254'), +('Carl', 'Martinez', 'cmartinez1k@geocities.jp', 'ALK-Abello, Inc.', '128.216.69.116'), +('Stephen', 'Anderson', 'sanderson1l@odnoklassniki.ru', 'Cardinal Health', '145.154.63.186'), +('Diana', 'Payne', 'dpayne1m@ftc.gov', 'Pharmaceutical Associates, Inc.', '98.9.155.136'), +('Judy', 'Gonzalez', 'jgonzalez1n@walmart.com', 'SHISEIDO CO., LTD.', '73.96.109.149'), +('Steve', 'Cole', 'scole1o@flickr.com', 'Walgreen Company', '251.244.20.117'), +('Johnny', 'Ellis', 'jellis1p@time.com', 'Jubilant HollisterStier LLC', '188.153.76.182'), +('Andrea', 'Hamilton', 'ahamilton1q@dailymail.co.uk', 'ALK-Abello, Inc.', '229.58.149.141'), +('Sean', 'Kennedy', 'skennedy1r@nifty.com', 'Newton Laboratories, Inc.', '227.105.251.134'), +('Sara', 'Grant', 'sgrant1s@flickr.com', 'Rubbermaid Commercial Products LLC', '96.211.162.73'), +('Joan', 'Bennett', 'jbennett1t@forbes.com', 'Nelco Laboratories, Inc.', '143.27.240.163'), +('Judith', 'Daniels', 'jdaniels1u@theguardian.com', 'Newton Laboratories, Inc.', '164.99.249.153'), +('Irene', 'Bennett', 'ibennett1v@comsenz.com', 'Cellab Co., Ltd.', '112.104.12.122'), +('Katherine', 'Perez', 'kperez1w@phpbb.com', 'Temple Industrial Welding Supply Co', '211.31.214.131'), +('Jean', 'Kim', 'jkim1x@umich.edu', 'Bryant Ranch Prepack', '245.252.150.110'), +('Walter', 'Hernandez', 'whernandez1y@nbcnews.com', 'Virtus Pharmaceuticals LLC', '200.201.83.21'), +('Larry', 'Scott', 'lscott1z@quantcast.com', 'BIOKEY INC.', '122.141.109.98'), +('Gerald', 'Palmer', 'gpalmer20@usgs.gov', 'JAFRA COSMETICS INTERNATIONAL', '60.173.159.145'), +('Harry', 'Andrews', 'handrews21@alexa.com', 'NCS HealthCare of KY, Inc dba Vangard Labs', '210.64.37.91'), +('Jerry', 'Morrison', 'jmorrison22@drupal.org', 'Teva Pharmaceuticals USA Inc', '83.190.174.61'), +('Irene', 'Diaz', 'idiaz23@joomla.org', 'Dolgencorp, LLC', '214.16.44.235'), +('Brenda', 'Hansen', 'bhansen24@wisc.edu', 'REMEDYREPACK INC.', '167.231.200.232'), +('Carlos', 'Williamson', 'cwilliamson25@w3.org', 'Kroger Company', '251.202.210.204'), +('David', 'Fuller', 'dfuller26@canalblog.com', 'Supervalu Inc', '175.125.205.131'), +('Norma', 'Bishop', 'nbishop27@jugem.jp', 'Mylan Institutional Inc.', '208.162.25.149'), +('Brenda', 'Daniels', 'bdaniels28@mediafire.com', 'Space Brands Limited', '92.235.250.138'), +('Kathy', 'Reed', 'kreed29@prweb.com', 'Rugby Laboratories Inc.', '182.114.174.63'), +('Anthony', 'Long', 'along2a@dropbox.com', 'Fresenius Kabi USA, LLC', '160.146.121.173'), +('Craig', 'Palmer', 'cpalmer2b@desdev.cn', 'Bio-Pharm, Inc.', '135.77.134.24'), +('Rachel', 'Banks', 'rbanks2c@devhub.com', 'Sam''s West Inc', '35.72.5.193'), +('Kenneth', 'Peters', 'kpeters2d@ocn.ne.jp', 'International Labs, Inc.', '11.38.191.65'), +('Susan', 'Clark', 'sclark2e@ed.gov', 'Shionogi Inc.', '19.243.67.80'), +('Walter', 'Sullivan', 'wsullivan2f@vinaora.com', 'STAT Rx USA LLC', '154.137.170.227'), +('Kathleen', 'Wood', 'kwood2g@salon.com', 'Freds Inc', '155.54.131.149'), +('Phyllis', 'Henderson', 'phenderson2h@walmart.com', 'REMEDYREPACK INC.', '146.65.150.251'), +('Cheryl', 'Wells', 'cwells2i@gov.uk', 'Rebel Distributors Corp', '69.127.148.31'), +('Rose', 'Bradley', 'rbradley2j@un.org', 'Hi-Tech Pharmacal Co., Inc.', '150.101.165.102'), +('Aaron', 'Moreno', 'amoreno2k@tinypic.com', 'Pharmacia and Upjohn Company', '50.27.226.40'), +('Amy', 'Campbell', 'acampbell2l@auda.org.au', 'Chi Research, Inc.', '242.64.63.241'), +('Rebecca', 'Butler', 'rbutler2m@godaddy.com', 'Cardinal Health', '40.55.159.66'), +('Justin', 'Rodriguez', 'jrodriguez2n@meetup.com', 'Hikma Pharmaceutical', '118.9.132.156'), +('Donald', 'Nelson', 'dnelson2o@narod.ru', 'Nature''s Way Products, Inc.', '165.174.28.134'), +('Edward', 'Lawson', 'elawson2p@addtoany.com', 'Apotheca Company', '135.17.238.170'), +('Paul', 'Bell', 'pbell2q@simplemachines.org', 'Washington Homeopathic Products', '235.149.137.62'), +('Mark', 'Rose', 'mrose2r@google.pl', 'AMERICAN SALES COMPANY', '164.108.170.187'); diff --git a/test/integration/003_simple_reference_test/seed.sql b/test/integration/003_simple_reference_test/seed.sql index 94a2ffd96db..f5e7208e26e 100644 --- a/test/integration/003_simple_reference_test/seed.sql +++ b/test/integration/003_simple_reference_test/seed.sql @@ -1,12 +1,13 @@ -create table simple_reference_003.summary_expected ( +create table "simple_reference_003"."summary_expected" ( gender VARCHAR(10), ct BIGINT ); -insert into simple_reference_003.summary_expected (gender, ct) values ('Female', 40); -insert into simple_reference_003.summary_expected (gender, ct) values ('Male', 60); +insert into "simple_reference_003"."summary_expected" (gender, ct) values +('Female', 40), +('Male', 60); -create table simple_reference_003.seed ( +create table "simple_reference_003"."seed" ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -15,104 +16,104 @@ create table simple_reference_003.seed ( ip_address VARCHAR(20) ); - -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'); -insert into simple_reference_003.seed (first_name, last_name, email, gender, ip_address) values ('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); +insert into "simple_reference_003"."seed" (first_name, last_name, email, gender, ip_address) values +('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), +('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), +('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), +('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), +('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), +('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), +('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), +('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), +('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), +('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), +('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), +('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), +('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), +('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), +('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), +('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), +('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), +('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), +('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), +('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), +('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), +('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), +('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), +('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), +('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), +('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), +('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), +('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), +('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), +('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), +('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), +('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), +('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), +('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), +('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), +('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), +('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), +('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), +('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), +('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), +('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), +('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), +('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), +('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), +('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), +('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), +('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), +('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), +('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), +('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), +('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), +('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), +('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), +('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), +('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), +('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), +('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), +('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), +('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), +('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), +('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), +('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), +('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), +('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), +('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), +('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), +('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), +('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), +('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), +('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), +('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), +('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), +('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), +('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), +('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), +('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), +('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), +('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), +('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), +('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), +('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), +('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), +('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), +('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), +('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), +('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), +('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), +('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), +('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), +('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), +('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), +('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), +('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), +('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), +('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), +('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), +('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), +('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), +('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), +('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); diff --git a/test/integration/003_simple_reference_test/test_simple_reference.py b/test/integration/003_simple_reference_test/test_simple_reference.py index f9ea7ce61a6..73eefb0e236 100644 --- a/test/integration/003_simple_reference_test/test_simple_reference.py +++ b/test/integration/003_simple_reference_test/test_simple_reference.py @@ -1,11 +1,10 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestSimpleReference(DBTIntegrationTest): def setUp(self): - DBTIntegrationTest.setUp(self) - - self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + pass @property def schema(self): @@ -15,7 +14,46 @@ def schema(self): def models(self): return "test/integration/003_simple_reference_test/models" - def test_simple_reference(self): + @attr(type='postgres') + def test__postgres__simple_reference(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + + self.run_dbt() + + # Copies should match + self.assertTablesEqual("seed","incremental_copy") + self.assertTablesEqual("seed","materialized_copy") + self.assertTablesEqual("seed","view_copy") + + # Summaries should match + self.assertTablesEqual("summary_expected","incremental_summary") + self.assertTablesEqual("summary_expected","materialized_summary") + self.assertTablesEqual("summary_expected","view_summary") + self.assertTablesEqual("summary_expected","ephemeral_summary") + + self.run_sql_file("test/integration/003_simple_reference_test/update.sql") + + self.run_dbt() + + # Copies should match + self.assertTablesEqual("seed","incremental_copy") + self.assertTablesEqual("seed","materialized_copy") + self.assertTablesEqual("seed","view_copy") + + # Summaries should match + self.assertTablesEqual("summary_expected","incremental_summary") + self.assertTablesEqual("summary_expected","materialized_summary") + self.assertTablesEqual("summary_expected","view_summary") + self.assertTablesEqual("summary_expected","ephemeral_summary") + + @attr(type='snowflake') + def test__snowflake__simple_reference(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + self.run_dbt() # Copies should match @@ -44,7 +82,48 @@ def test_simple_reference(self): self.assertTablesEqual("summary_expected","view_summary") self.assertTablesEqual("summary_expected","ephemeral_summary") - def test_simple_reference_with_models(self): + @attr(type='postgres') + def test__postgres__simple_reference_with_models(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + + # Run materialized_copy, ephemeral_copy, and their dependents + # ephemeral_copy should not actually be materialized b/c it is ephemeral + # the dependent ephemeral_summary, however, should be materialized as a table + self.run_dbt(['run', '--models', 'materialized_copy', 'ephemeral_copy']) + + # Copies should match + self.assertTablesEqual("seed","materialized_copy") + + # Summaries should match + self.assertTablesEqual("summary_expected","materialized_summary") + self.assertTablesEqual("summary_expected","ephemeral_summary") + + created_models = self.get_models_in_schema() + + self.assertFalse('incremental_copy' in created_models) + self.assertFalse('incremental_summary' in created_models) + self.assertFalse('view_copy' in created_models) + self.assertFalse('view_summary' in created_models) + + # make sure this wasn't errantly materialized + self.assertFalse('ephemeral_copy' in created_models) + + self.assertTrue('materialized_copy' in created_models) + self.assertTrue('materialized_summary' in created_models) + self.assertEqual(created_models['materialized_copy'], 'table') + self.assertEqual(created_models['materialized_summary'], 'table') + + self.assertTrue('ephemeral_summary' in created_models) + self.assertEqual(created_models['ephemeral_summary'], 'table') + + @attr(type='snowflake') + def test__snowflake__simple_reference_with_models(self): + self.use_default_project() + self.use_profile('snowflake') + self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + # Run materialized_copy, ephemeral_copy, and their dependents # ephemeral_copy should not actually be materialized b/c it is ephemeral # the dependent ephemeral_summary, however, should be materialized as a table diff --git a/test/integration/003_simple_reference_test/update.sql b/test/integration/003_simple_reference_test/update.sql index 182757f1e66..36eb85c7d5e 100644 --- a/test/integration/003_simple_reference_test/update.sql +++ b/test/integration/003_simple_reference_test/update.sql @@ -1,100 +1,101 @@ -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'); -insert into simple_copy_001.seed (first_name, last_name, email, gender, ip_address) values ('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117'); +insert into "simple_copy_001"."seed" (first_name, last_name, email, gender, ip_address) values +('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), +('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), +('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), +('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'), +('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'), +('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'), +('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'), +('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'), +('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'), +('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'), +('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'), +('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'), +('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'), +('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'), +('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'), +('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'), +('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'), +('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'), +('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'), +('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'), +('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'), +('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'), +('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'), +('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'), +('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'), +('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'), +('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'), +('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'), +('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'), +('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'), +('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'), +('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'), +('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'), +('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'), +('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'), +('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'), +('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'), +('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'), +('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'), +('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'), +('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'), +('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'), +('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'), +('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'), +('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'), +('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'), +('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'), +('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'), +('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'), +('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'), +('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'), +('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'), +('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'), +('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'), +('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'), +('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'), +('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'), +('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'), +('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'), +('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'), +('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'), +('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'), +('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'), +('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'), +('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'), +('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'), +('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'), +('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'), +('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'), +('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'), +('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'), +('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'), +('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'), +('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'), +('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'), +('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'), +('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'), +('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'), +('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'), +('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'), +('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'), +('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'), +('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'), +('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'), +('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'), +('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'), +('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'), +('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'), +('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'), +('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'), +('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'), +('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'), +('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'), +('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'), +('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'), +('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'), +('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'), +('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'), +('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'), +('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117'); diff --git a/test/integration/004_simple_archive_test/invalidate_postgres.sql b/test/integration/004_simple_archive_test/invalidate_postgres.sql new file mode 100644 index 00000000000..3c46ebdb276 --- /dev/null +++ b/test/integration/004_simple_archive_test/invalidate_postgres.sql @@ -0,0 +1,12 @@ + +-- update records 11 - 21. Change email and updated_at field +update "simple_archive_004"."seed" set + "updated_at" = "updated_at" + interval '1 hour', + "email" = 'new_' || "email" +where "id" >= 10 and "id" <= 20; + + +-- invalidate records 11 - 21 +update "simple_archive_004"."archive_expected" set + "valid_to" = "updated_at" + interval '1 hour' +where "id" >= 10 and "id" <= 20; diff --git a/test/integration/004_simple_archive_test/invalidate_snowflake.sql b/test/integration/004_simple_archive_test/invalidate_snowflake.sql new file mode 100644 index 00000000000..6484b25de56 --- /dev/null +++ b/test/integration/004_simple_archive_test/invalidate_snowflake.sql @@ -0,0 +1,12 @@ + +-- update records 11 - 21. Change email and updated_at field +update "simple_archive_004"."seed" set + "updated_at" = DATEADD(hour, 1, "updated_at"), + "email" = 'new_' || "email" +where "id" >= 10 and "id" <= 20; + + +-- invalidate records 11 - 21 +update "simple_archive_004"."archive_expected" set + "valid_to" = DATEADD(hour, 1, "updated_at") +where "id" >= 10 and "id" <= 20; diff --git a/test/integration/004_simple_archive_test/seed.sql b/test/integration/004_simple_archive_test/seed.sql index 908d1bc32e4..7053ee7c67d 100644 --- a/test/integration/004_simple_archive_test/seed.sql +++ b/test/integration/004_simple_archive_test/seed.sql @@ -1,79 +1,80 @@ -create table simple_archive_004.seed ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), - updated_at TIMESTAMP WITHOUT TIME ZONE +create table "simple_archive_004"."seed" ( + "id" INTEGER, + "first_name" VARCHAR(50), + "last_name" VARCHAR(50), + "email" VARCHAR(50), + "gender" VARCHAR(50), + "ip_address" VARCHAR(20), + "updated_at" TIMESTAMP WITHOUT TIME ZONE ); -create table simple_archive_004.archive_expected ( - id INTEGER, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20), +create table "simple_archive_004"."archive_expected" ( + "id" INTEGER, + "first_name" VARCHAR(50), + "last_name" VARCHAR(50), + "email" VARCHAR(50), + "gender" VARCHAR(50), + "ip_address" VARCHAR(20), -- archival fields - updated_at TIMESTAMP WITHOUT TIME ZONE, - valid_from TIMESTAMP WITHOUT TIME ZONE, - valid_to TIMESTAMP WITHOUT TIME ZONE, - scd_id VARCHAR(255), - dbt_updated_at TIMESTAMP WITHOUT TIME ZONE + "updated_at" TIMESTAMP WITHOUT TIME ZONE, + "valid_from" TIMESTAMP WITHOUT TIME ZONE, + "valid_to" TIMESTAMP WITHOUT TIME ZONE, + "scd_id" VARCHAR(255), + "dbt_updated_at" TIMESTAMP WITHOUT TIME ZONE ); -- seed inserts -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (20, 'Phyllis', 'Fox', 'pfoxj@creativecommons.org', 'Female', '163.191.232.95', '2016-08-21 10:35:19'); +insert into "simple_archive_004"."seed" ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +(1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), +(2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), +(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), +(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), +(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), +(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), +(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), +(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), +(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), +(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), +(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), +(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), +(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), +(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), +(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), +(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), +(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), +(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), +(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), +(20, 'Phyllis', 'Fox', 'pfoxj@creativecommons.org', 'Female', '163.191.232.95', '2016-08-21 10:35:19'); -- populate archive table -insert into simple_archive_004.archive_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - valid_from, - valid_to, - dbt_updated_at, - scd_id +insert into "simple_archive_004"."archive_expected" ( + "id", + "first_name", + "last_name", + "email", + "gender", + "ip_address", + "updated_at", + "valid_from", + "valid_to", + "dbt_updated_at", + "scd_id" ) select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, + "id", + "first_name", + "last_name", + "email", + "gender", + "ip_address", + "updated_at", -- fields added by archival - updated_at as valid_from, + "updated_at" as valid_from, null::timestamp as valid_to, - updated_at as dbt_updated_at, - md5(id || '|' || updated_at::text) as scd_id -from simple_archive_004.seed; + "updated_at" as dbt_updated_at, + md5("id" || '|' || "updated_at"::text) as scd_id +from "simple_archive_004"."seed"; diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py index 0a4f7d4c4e1..87de009fbe5 100644 --- a/test/integration/004_simple_archive_test/test_simple_archive.py +++ b/test/integration/004_simple_archive_test/test_simple_archive.py @@ -1,11 +1,10 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestSimpleArchive(DBTIntegrationTest): def setUp(self): - DBTIntegrationTest.setUp(self) - - self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") + pass @property def schema(self): @@ -34,11 +33,34 @@ def project_config(self): ] } - def test_simple_dependency(self): + @attr(type='postgres') + def test__postgres__simple_archive(self): + self.use_default_project() + self.use_profile('postgres') + self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") + + self.run_dbt(["archive"]) + + self.assertTablesEqual("archive_expected","archive_actual") + + self.run_sql_file("test/integration/004_simple_archive_test/invalidate_postgres.sql") + self.run_sql_file("test/integration/004_simple_archive_test/update.sql") + + self.run_dbt(["archive"]) + + self.assertTablesEqual("archive_expected","archive_actual") + + @attr(type='snowflake') + def test__snowflake__simple_archive(self): + self.use_default_project() + self.use_profile('snowflake') + self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") + self.run_dbt(["archive"]) self.assertTablesEqual("archive_expected","archive_actual") + self.run_sql_file("test/integration/004_simple_archive_test/invalidate_snowflake.sql") self.run_sql_file("test/integration/004_simple_archive_test/update.sql") self.run_dbt(["archive"]) diff --git a/test/integration/004_simple_archive_test/update.sql b/test/integration/004_simple_archive_test/update.sql index d8fa95ece0f..71927b2ea1d 100644 --- a/test/integration/004_simple_archive_test/update.sql +++ b/test/integration/004_simple_archive_test/update.sql @@ -1,89 +1,77 @@ - --- update records 11 - 21. Change email and updated_at field -update simple_archive_004.seed set - updated_at = updated_at + interval '1 hour', - email = 'new_' || email -where id >= 10 and id <= 20; - - --- invalidate records 11 - 21 -update simple_archive_004.archive_expected set - valid_to = updated_at + interval '1 hour' -where id >= 10 and id <= 20; - -- insert v2 of the 11 - 21 records -insert into simple_archive_004.archive_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - valid_from, - valid_to, - dbt_updated_at, - scd_id +insert into "simple_archive_004"."archive_expected" ( + "id", + "first_name", + "last_name", + "email", + "gender", + "ip_address", + "updated_at", + "valid_from", + "valid_to", + "dbt_updated_at", + "scd_id" ) select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, + "id", + "first_name", + "last_name", + "email", + "gender", + "ip_address", + "updated_at", -- fields added by archival - updated_at as valid_from, - null::timestamp as valid_to, - updated_at as dbt_updated_at, - md5(id || '|' || updated_at::text) as scd_id -from simple_archive_004.seed -where id >= 10 and id <= 20; + "updated_at" as "valid_from", + null::timestamp as "valid_to", + "updated_at" as "dbt_updated_at", + md5("id" || '|' || "updated_at"::text) as "scd_id" +from "simple_archive_004"."seed" +where "id" >= 10 and "id" <= 20; -- insert 10 new records -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (24, 'William', 'Watkins', 'wwatkinsn@guardian.co.uk', 'Male', '78.155.84.6', '2016-03-08 19:13:08'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (25, 'Judy', 'Cooper', 'jcoopero@google.com.au', 'Female', '24.149.123.184', '2016-10-05 20:49:33'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (26, 'Shirley', 'Castillo', 'scastillop@samsung.com', 'Female', '129.252.181.12', '2016-06-20 21:12:21'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (27, 'Justin', 'Harper', 'jharperq@opera.com', 'Male', '131.172.103.218', '2016-05-21 22:56:46'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (28, 'Marie', 'Medina', 'mmedinar@nhs.uk', 'Female', '188.119.125.67', '2015-10-08 13:44:33'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (29, 'Kelly', 'Edwards', 'kedwardss@phoca.cz', 'Female', '47.121.157.66', '2015-09-15 06:33:37'); -insert into simple_archive_004.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (30, 'Carl', 'Coleman', 'ccolemant@wikipedia.org', 'Male', '82.227.154.83', '2016-05-26 16:46:40'); +insert into "simple_archive_004"."seed" ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +(21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), +(22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), +(23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), +(24, 'William', 'Watkins', 'wwatkinsn@guardian.co.uk', 'Male', '78.155.84.6', '2016-03-08 19:13:08'), +(25, 'Judy', 'Cooper', 'jcoopero@google.com.au', 'Female', '24.149.123.184', '2016-10-05 20:49:33'), +(26, 'Shirley', 'Castillo', 'scastillop@samsung.com', 'Female', '129.252.181.12', '2016-06-20 21:12:21'), +(27, 'Justin', 'Harper', 'jharperq@opera.com', 'Male', '131.172.103.218', '2016-05-21 22:56:46'), +(28, 'Marie', 'Medina', 'mmedinar@nhs.uk', 'Female', '188.119.125.67', '2015-10-08 13:44:33'), +(29, 'Kelly', 'Edwards', 'kedwardss@phoca.cz', 'Female', '47.121.157.66', '2015-09-15 06:33:37'), +(30, 'Carl', 'Coleman', 'ccolemant@wikipedia.org', 'Male', '82.227.154.83', '2016-05-26 16:46:40'); -- add these new records to the archive table -insert into simple_archive_004.archive_expected ( - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, - valid_from, - valid_to, - dbt_updated_at, - scd_id +insert into "simple_archive_004"."archive_expected" ( + "id", + "first_name", + "last_name", + "email", + "gender", + "ip_address", + "updated_at", + "valid_from", + "valid_to", + "dbt_updated_at", + "scd_id" ) select - id, - first_name, - last_name, - email, - gender, - ip_address, - updated_at, + "id", + "first_name", + "last_name", + "email", + "gender", + "ip_address", + "updated_at", -- fields added by archival - updated_at as valid_from, - null::timestamp as valid_to, - updated_at as dbt_updated_at, - md5(id || '|' || updated_at::text) as scd_id -from simple_archive_004.seed -where id > 20; + "updated_at" as "valid_from", + null::timestamp as "valid_to", + "updated_at" as "dbt_updated_at", + md5("id" || '|' || "updated_at"::text) as "scd_id" +from "simple_archive_004"."seed" +where "id" > 20; diff --git a/test/integration/005_simple_seed_test/test_simple_seed.py b/test/integration/005_simple_seed_test/test_simple_seed.py index 9bfb3dc712f..2aedc2d09fc 100644 --- a/test/integration/005_simple_seed_test/test_simple_seed.py +++ b/test/integration/005_simple_seed_test/test_simple_seed.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestSimpleSeed(DBTIntegrationTest): @@ -21,6 +22,7 @@ def project_config(self): "data-paths": ['test/integration/005_simple_seed_test/data'] } + @attr(type='postgres') def test_simple_seed(self): self.run_dbt(["seed"]) self.assertTablesEqual("seed_actual","seed_expected") @@ -30,6 +32,7 @@ def test_simple_seed(self): self.assertTablesEqual("seed_actual","seed_expected") + @attr(type='postgres') def test_simple_seed_with_drop(self): self.run_dbt(["seed"]) self.assertTablesEqual("seed_actual","seed_expected") diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency.py b/test/integration/006_simple_dependency_test/test_simple_dependency.py index 7ee2385a152..d42b4a6d6d5 100644 --- a/test/integration/006_simple_dependency_test/test_simple_dependency.py +++ b/test/integration/006_simple_dependency_test/test_simple_dependency.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestSimpleDependency(DBTIntegrationTest): @@ -22,6 +23,7 @@ def project_config(self): ] } + @attr(type='postgres') def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) @@ -41,6 +43,7 @@ def test_simple_dependency(self): self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") + @attr(type='postgres') def test_simple_dependency_with_models(self): self.run_dbt(["deps"]) self.run_dbt(["run", '--models', 'view']) @@ -93,6 +96,7 @@ def deps_run_assert_equality(self): self.assertEqual(created_models['view_summary'], 'view') self.assertEqual(created_models['incremental'], 'table') + @attr(type='postgres') def test_simple_dependency(self): self.deps_run_assert_equality() @@ -102,6 +106,7 @@ def test_simple_dependency(self): self.deps_run_assert_equality() + @attr(type='postgres') def test_empty_models_not_compiled_in_dependencies(self): self.deps_run_assert_equality() diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py b/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py index 53b98b245aa..2d4daaf7ef4 100644 --- a/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py +++ b/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class BaseTestSimpleDependencyWithConfigs(DBTIntegrationTest): @@ -32,7 +33,7 @@ def project_config(self): ] } - + @attr(type='postgres') def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) @@ -66,6 +67,7 @@ def project_config(self): } + @attr(type='postgres') def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) @@ -102,6 +104,7 @@ def project_config(self): } + @attr(type='postgres') def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) @@ -146,6 +149,7 @@ def project_config(self): } + @attr(type='postgres') def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) diff --git a/test/integration/007_dry_run_test/test_dry_run.py b/test/integration/007_dry_run_test/test_dry_run.py index c9770fcf089..92846586e78 100644 --- a/test/integration/007_dry_run_test/test_dry_run.py +++ b/test/integration/007_dry_run_test/test_dry_run.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestDryRun(DBTIntegrationTest): @@ -14,6 +15,7 @@ def schema(self): def models(self): return "test/integration/007_dry_run_test/models" + @attr(type='postgres') def test_dry_run(self): self.run_dbt(["run", '--dry']) diff --git a/test/integration/008_schema_tests_test/test_schema_tests.py b/test/integration/008_schema_tests_test/test_schema_tests.py index 49c9732afee..df8be4fd8a2 100644 --- a/test/integration/008_schema_tests_test/test_schema_tests.py +++ b/test/integration/008_schema_tests_test/test_schema_tests.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest from dbt.task.test import TestTask @@ -16,7 +17,7 @@ def setUp(self): DBTIntegrationTest.setUp(self) self.run_sql_file("test/integration/008_schema_tests_test/seed.sql") self.run_sql_file("test/integration/008_schema_tests_test/seed_failure.sql") - + @property def schema(self): return "schema_tests_008" @@ -32,6 +33,7 @@ def run_schema_validations(self): test_task = TestTask(args, project) return test_task.run() + @attr(type='postgres') def test_schema_tests(self): self.run_dbt() test_results = self.run_schema_validations() @@ -71,6 +73,7 @@ def run_schema_validations(self): test_task = TestTask(args, project) return test_task.run() + @attr(type='postgres') def test_malformed_schema_test_wont_brick_run(self): # dbt run should work (Despite broken schema test) self.run_dbt() diff --git a/test/integration/009_data_tests_test/test_data_tests.py b/test/integration/009_data_tests_test/test_data_tests.py index b2110bc0390..dc579f30fd3 100644 --- a/test/integration/009_data_tests_test/test_data_tests.py +++ b/test/integration/009_data_tests_test/test_data_tests.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest from dbt.task.test import TestTask @@ -15,7 +16,7 @@ class TestDataTests(DBTIntegrationTest): def setUp(self): DBTIntegrationTest.setUp(self) self.run_sql_file("test/integration/009_data_tests_test/seed.sql") - + @property def project_config(self): return { @@ -37,6 +38,7 @@ def run_data_validations(self): test_task = TestTask(args, project) return test_task.run() + @attr(type='postgres') def test_data_tests(self): self.run_dbt() test_results = self.run_data_validations() diff --git a/test/integration/010_permission_tests/test_permissions.py b/test/integration/010_permission_tests/test_permissions.py index 8439fcc4311..a322457e596 100644 --- a/test/integration/010_permission_tests/test_permissions.py +++ b/test/integration/010_permission_tests/test_permissions.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestPermissions(DBTIntegrationTest): @@ -20,11 +21,17 @@ def schema(self): def models(self): return "test/integration/010_permission_tests/models" + @attr(type='postgres') def test_read_permissions(self): failed = False # run model as the noaccess user # this will fail with a RuntimeError, which should be caught by the dbt runner - self.run_dbt(['run', '--target', 'noaccess']) + # it's not, wrapping this for now + # TODO handle RuntimeErrors for connection failure + try: + self.run_dbt(['run', '--target', 'noaccess']) + except: + pass diff --git a/test/integration/011_invalid_model_tests/test_invalid_models.py b/test/integration/011_invalid_model_tests/test_invalid_models.py index 14e5563e5a9..0f8338a4ad9 100644 --- a/test/integration/011_invalid_model_tests/test_invalid_models.py +++ b/test/integration/011_invalid_model_tests/test_invalid_models.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestInvalidViewModels(DBTIntegrationTest): @@ -15,6 +16,7 @@ def schema(self): def models(self): return "test/integration/011_invalid_model_tests/models" + @attr(type='postgres') def test_view_with_incremental_attributes(self): self.run_dbt() @@ -33,6 +35,7 @@ def schema(self): def models(self): return "test/integration/011_invalid_model_tests/models-2" + @attr(type='postgres') def test_view_with_incremental_attributes(self): try: @@ -57,6 +60,7 @@ def schema(self): def models(self): return "test/integration/011_invalid_model_tests/models-3" + @attr(type='postgres') def test_view_with_incremental_attributes(self): try: diff --git a/test/integration/012_profile_config_tests/test_profile_config.py b/test/integration/012_profile_config_tests/test_profile_config.py index a22edaa439f..59d8e9f52e3 100644 --- a/test/integration/012_profile_config_tests/test_profile_config.py +++ b/test/integration/012_profile_config_tests/test_profile_config.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest import dbt.deprecations @@ -37,6 +38,7 @@ def profile_config(self): } } + @attr(type='postgres') def test_deprecated_run_target_config(self): self.run_dbt() @@ -81,6 +83,7 @@ def profile_config(self): } } + @attr(type='postgres') def test_deprecated_run_target_config(self): self.run_dbt() diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index 84bbfd5f459..514a0b54540 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestContextVars(DBTIntegrationTest): @@ -65,11 +66,12 @@ def get_ctx_vars(self): field_list = ", ".join(['"{}"'.format(f) for f in self.fields]) query = 'select {field_list} from {schema}.context'.format(field_list=field_list, schema=self.schema) - vals = self.run_sql(query) + vals = self.run_sql(query, fetch='all') ctx = dict([(k,v) for (k,v) in zip(self.fields, vals[0])]) return ctx + @attr(type='postgres') def test_env_vars_dev(self): self.run_dbt(['run']) ctx = self.get_ctx_vars() @@ -89,6 +91,7 @@ def test_env_vars_dev(self): self.assertEqual(ctx['target.user'], 'root') self.assertEqual(ctx['target.pass'], '') + @attr(type='postgres') def test_env_vars_prod(self): self.run_dbt(['run', '--target', 'prod']) ctx = self.get_ctx_vars() @@ -107,4 +110,3 @@ def test_env_vars_prod(self): self.assertEqual(ctx['target.type'], 'postgres') self.assertEqual(ctx['target.user'], 'root') self.assertEqual(ctx['target.pass'], '') - diff --git a/test/integration/014_pre_post_run_hook_tests/test_pre_post_run_hooks.py b/test/integration/014_pre_post_run_hook_tests/test_pre_post_run_hooks.py index eb85f8aeffb..226dc9ef3df 100644 --- a/test/integration/014_pre_post_run_hook_tests/test_pre_post_run_hooks.py +++ b/test/integration/014_pre_post_run_hook_tests/test_pre_post_run_hooks.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest @@ -102,7 +103,7 @@ def get_ctx_vars(self, state): field_list = ", ".join(['"{}"'.format(f) for f in self.fields]) query = "select {field_list} from {schema}.on_run_hook where state = '{state}'".format(field_list=field_list, schema=self.schema, state=state) - vals = self.run_sql(query) + vals = self.run_sql(query, fetch='all') self.assertFalse(len(vals) == 0, 'nothing inserted into on_run_hook table') ctx = dict([(k,v) for (k,v) in zip(self.fields, vals[0])]) @@ -125,7 +126,7 @@ def check_hooks(self, state): self.assertTrue(ctx['run_started_at'] is not None and len(ctx['run_started_at']) > 0, 'run_started_at was not set') self.assertTrue(ctx['invocation_id'] is not None and len(ctx['invocation_id']) > 0, 'invocation_id was not set') - + @attr(type='postgres') def test_pre_and_post_run_hooks(self): self.run_dbt(['run']) diff --git a/test/integration/015_cli_invocation_tests/test_cli_invocation.py b/test/integration/015_cli_invocation_tests/test_cli_invocation.py index 45e7360526d..e7782cbb795 100644 --- a/test/integration/015_cli_invocation_tests/test_cli_invocation.py +++ b/test/integration/015_cli_invocation_tests/test_cli_invocation.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest, DBT_PROFILES import os, shutil, yaml @@ -16,10 +17,12 @@ def schema(self): def models(self): return "test/integration/015_cli_invocation_tests/models" + @attr(type='postgres') def test_toplevel_dbt_run(self): self.run_dbt(['run']) self.assertTablesEqual("seed", "model") + @attr(type='postgres') def test_subdir_dbt_run(self): os.chdir(os.path.join(self.models, "subdir1")) @@ -81,6 +84,7 @@ def custom_schema(self): def models(self): return "test/integration/015_cli_invocation_tests/models" + @attr(type='postgres') def test_toplevel_dbt_run_with_profile_dir_arg(self): self.run_dbt(['run', '--profiles-dir', 'dbt-profile']) @@ -94,4 +98,3 @@ def test_toplevel_dbt_run_with_profile_dir_arg(self): # make sure the test runs against `custom_schema` for test_result in res: self.assertTrue(self.custom_schema, test_result.model.compiled_contents) - diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index 770ff625006..01977641f16 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestMacros(DBTIntegrationTest): @@ -23,6 +24,7 @@ def project_config(self): ] } + @attr(type='postgres') def test_working_macros(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) @@ -50,6 +52,7 @@ def project_config(self): "macro-paths": ["test/integration/016_macro_tests/bad-macros"] } + @attr(type='postgres') def test_invalid_macro(self): try: @@ -80,6 +83,7 @@ def project_config(self): ] } + @attr(type='postgres') def test_working_macros(self): self.run_dbt(["deps"]) diff --git a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py index b3f94db7823..30c051be181 100644 --- a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py +++ b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py @@ -1,3 +1,4 @@ +from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest class TestRuntimeMaterialization(DBTIntegrationTest): @@ -15,6 +16,7 @@ def schema(self): def models(self): return "test/integration/017_runtime_materialization_tests/models" + @attr(type='postgres') def test_full_refresh(self): self.run_dbt(['run', '--full-refresh']) @@ -30,6 +32,7 @@ def test_full_refresh(self): self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") + @attr(type='postgres') def test_non_destructive(self): self.run_dbt(['run', '--non-destructive']) @@ -45,6 +48,7 @@ def test_non_destructive(self): self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") + @attr(type='postgres') def test_full_refresh_and_non_destructive(self): self.run_dbt(['run', '--full-refresh', '--non-destructive']) @@ -59,4 +63,3 @@ def test_full_refresh_and_non_destructive(self): self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - diff --git a/test/integration/base.py b/test/integration/base.py index 17b1540afe8..c019a642850 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -4,35 +4,15 @@ import yaml import time -from test.integration.connection import handle +from dbt.adapters.factory import get_adapter DBT_CONFIG_DIR = os.path.expanduser(os.environ.get("DBT_CONFIG_DIR", '/root/.dbt')) DBT_PROFILES = os.path.join(DBT_CONFIG_DIR, 'profiles.yml') class DBTIntegrationTest(unittest.TestCase): - def setUp(self): - # create a dbt_project.yml - - base_project_config = { - 'name': 'test', - 'version': '1.0', - 'test-paths': [], - 'source-paths': [self.models], - 'profile': 'test' - } - - project_config = {} - project_config.update(base_project_config) - project_config.update(self.project_config) - - with open("dbt_project.yml", 'w') as f: - yaml.safe_dump(project_config, f, default_flow_style=True) - - # create profiles - - profile_config = {} - default_profile_config = { + def postgres_profile(self): + return { 'config': { 'send_anonymous_usage_stats': False }, @@ -59,9 +39,118 @@ def setUp(self): 'schema': self.schema } }, - 'run-target': 'default2' + 'target': 'default2' + } + } + + def snowflake_profile(self): + return { + 'config': { + 'send_anonymous_usage_stats': False + }, + 'test': { + 'outputs': { + 'default2': { + 'type': 'snowflake', + 'threads': 1, + 'account': os.getenv('SNOWFLAKE_TEST_ACCOUNT'), + 'user': os.getenv('SNOWFLAKE_TEST_USER'), + 'password': os.getenv('SNOWFLAKE_TEST_PASSWORD'), + 'database': os.getenv('SNOWFLAKE_TEST_DATABASE'), + 'schema': self.schema, + 'warehouse': os.getenv('SNOWFLAKE_TEST_WAREHOUSE'), + }, + 'noaccess': { + 'type': 'snowflake', + 'threads': 1, + 'account': os.getenv('SNOWFLAKE_TEST_ACCOUNT'), + 'user': 'noaccess', + 'password': 'password', + 'database': os.getenv('SNOWFLAKE_TEST_DATABASE'), + 'schema': self.schema, + 'warehouse': os.getenv('SNOWFLAKE_TEST_WAREHOUSE'), + } + }, + 'target': 'default2' } } + + def get_profile(self, adapter_type): + if adapter_type == 'postgres': + return self.postgres_profile() + elif adapter_type == 'snowflake': + return self.snowflake_profile() + + def setUp(self): + # create a dbt_project.yml + + base_project_config = { + 'name': 'test', + 'version': '1.0', + 'test-paths': [], + 'source-paths': [self.models], + 'profile': 'test' + } + + project_config = {} + project_config.update(base_project_config) + project_config.update(self.project_config) + + with open("dbt_project.yml", 'w') as f: + yaml.safe_dump(project_config, f, default_flow_style=True) + + # create profiles + + profile_config = {} + default_profile_config = self.postgres_profile() + profile_config.update(default_profile_config) + profile_config.update(self.profile_config) + + if not os.path.exists(DBT_CONFIG_DIR): + os.makedirs(DBT_CONFIG_DIR) + + with open(DBT_PROFILES, 'w') as f: + yaml.safe_dump(profile_config, f, default_flow_style=True) + + target = profile_config.get('test').get('target') + + if target is None: + target = profile_config.get('test').get('run-target') + + profile = profile_config.get('test').get('outputs').get(target) + + adapter = get_adapter(profile) + + # it's important to use a different connection handle here so + # we don't look into an incomplete transaction + connection = adapter.acquire_connection(profile) + self.handle = connection.get('handle') + self.adapter_type = profile.get('type') + + self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.schema)) + self.run_sql('CREATE SCHEMA "{}"'.format(self.schema)) + + def use_default_project(self): + # create a dbt_project.yml + base_project_config = { + 'name': 'test', + 'version': '1.0', + 'test-paths': [], + 'source-paths': [self.models], + 'profile': 'test' + } + + project_config = {} + project_config.update(base_project_config) + project_config.update(self.project_config) + + with open("dbt_project.yml", 'w') as f: + yaml.safe_dump(project_config, f, default_flow_style=True) + + def use_profile(self, adapter_type): + profile_config = {} + default_profile_config = self.get_profile(adapter_type) + profile_config.update(default_profile_config) profile_config.update(self.profile_config) @@ -71,8 +160,17 @@ def setUp(self): with open(DBT_PROFILES, 'w') as f: yaml.safe_dump(profile_config, f, default_flow_style=True) - self.run_sql("DROP SCHEMA IF EXISTS {} CASCADE;".format(self.schema)) - self.run_sql("CREATE SCHEMA {};".format(self.schema)) + profile = profile_config.get('test').get('outputs').get('default2') + adapter = get_adapter(profile) + + # it's important to use a different connection handle here so + # we don't look into an incomplete transaction + connection = adapter.acquire_connection(profile) + self.handle = connection.get('handle') + self.adapter_type = profile.get('type') + + self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.schema)) + self.run_sql('CREATE SCHEMA "{}"'.format(self.schema)) def tearDown(self): os.remove(DBT_PROFILES) @@ -97,25 +195,44 @@ def run_dbt(self, args=None): if args is None: args = ["run"] + args = ["--strict"] + args + return dbt.handle(args) def run_sql_file(self, path): with open(path, 'r') as f: - return self.run_sql(f.read()) + statements = f.read().split(";") + for statement in statements: + self.run_sql(statement) + + # horrible hack to support snowflake for right now + def transform_sql(self, query): + to_return = query + + if self.adapter_type == 'snowflake': + to_return = to_return.replace("BIGSERIAL", "BIGINT AUTOINCREMENT") + + return to_return + + def run_sql(self, query, fetch='None'): + if query.strip() == "": + return - def run_sql(self, query, fetch='all'): - with handle.cursor() as cursor: + with self.handle.cursor() as cursor: try: - cursor.execute(query) - handle.commit() + cursor.execute(self.transform_sql(query)) + self.handle.commit() if fetch == 'one': - output = cursor.fetchone() + return cursor.fetchone() + elif fetch == 'all': + return cursor.fetchall() else: - output = cursor.fetchall() - return output + return except BaseException as e: - handle.rollback() + self.handle.rollback() + print(query) print(e) + raise e def get_table_columns(self, table): sql = """ @@ -125,7 +242,7 @@ def get_table_columns(self, table): and table_schema = '{}' order by column_name asc""" - result = self.run_sql(sql.format(table, self.schema)) + result = self.run_sql(sql.format(table, self.schema), fetch='all') return result @@ -141,7 +258,7 @@ def get_models_in_schema(self): order by table_name """ - result = self.run_sql(sql.format(self.schema)) + result = self.run_sql(sql.format(self.schema), fetch='all') return {model_name: materialization for (model_name, materialization) in result} @@ -150,16 +267,19 @@ def assertTablesEqual(self, table_a, table_b): self.assertTableRowCountsEqual(table_a, table_b) columns = self.get_table_columns(table_a) - columns_csv = ", ".join([record[0] for record in columns]) + columns_csv = ", ".join(['"{}"'.format(record[0]) + for record in columns]) table_sql = "SELECT {} FROM {}" sql = """ SELECT COUNT(*) FROM ( - (SELECT {columns} FROM {schema}.{table_a} EXCEPT SELECT {columns} FROM {schema}.{table_b}) + (SELECT {columns} FROM "{schema}"."{table_a}" EXCEPT + SELECT {columns} FROM "{schema}"."{table_b}") UNION ALL - (SELECT {columns} FROM {schema}.{table_b} EXCEPT SELECT {columns} FROM {schema}.{table_a}) - ) AS _""".format( + (SELECT {columns} FROM "{schema}"."{table_b}" EXCEPT + SELECT {columns} FROM "{schema}"."{table_a}") + ) AS a""".format( columns=columns_csv, schema=self.schema, table_a=table_a, @@ -171,12 +291,12 @@ def assertTablesEqual(self, table_a, table_b): self.assertEquals( result[0], 0, - "{} rows had mismatches." + sql ) def assertTableRowCountsEqual(self, table_a, table_b): - table_a_result = self.run_sql("SELECT COUNT(*) FROM {}.{}".format(self.schema, table_a), fetch='one') - table_b_result = self.run_sql("SELECT COUNT(*) FROM {}.{}".format(self.schema, table_b), fetch='one') + table_a_result = self.run_sql('SELECT COUNT(*) FROM "{}"."{}"'.format(self.schema, table_a), fetch='one') + table_b_result = self.run_sql('SELECT COUNT(*) FROM "{}"."{}"'.format(self.schema, table_b), fetch='one') self.assertEquals( table_a_result[0], diff --git a/test/setup.sh b/test/setup.sh index 1d3d1733763..557805beae6 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -6,5 +6,5 @@ mkvirtualenv dbt cd /usr/src/app -pip install -r requirements.txt +pip install -r requirements.txt pip install -r dev_requirements.txt diff --git a/test/setup_db.sh b/test/setup_db.sh index c09cb1e3aab..222f59a3915 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -1,3 +1,4 @@ +set -x createdb dbt psql -c "CREATE ROLE root WITH UNENCRYPTED PASSWORD 'password';" -U postgres @@ -7,3 +8,5 @@ psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root;" -U postgres psql -c "CREATE ROLE noaccess WITH UNENCRYPTED PASSWORD 'password' NOSUPERUSER;" -U postgres; psql -c "ALTER ROLE noaccess WITH LOGIN;" -U postgres psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO noaccess;" -U postgres; + +set +x diff --git a/test/unit/test_postgres_adapter.py b/test/unit/test_postgres_adapter.py new file mode 100644 index 00000000000..b8520c84885 --- /dev/null +++ b/test/unit/test_postgres_adapter.py @@ -0,0 +1,47 @@ +import unittest + +import dbt.flags as flags + +from dbt.adapters.postgres import PostgresAdapter +from dbt.exceptions import ValidationException +from dbt.logger import GLOBAL_LOGGER as logger + + +class TestPostgresAdapter(unittest.TestCase): + + def setUp(self): + flags.STRICT_MODE = True + + self.profile = { + 'dbname': 'postgres', + 'user': 'root', + 'host': 'database', + 'pass': 'password', + 'port': 5432, + 'schema': 'public' + } + + def test_acquire_connection_validations(self): + try: + connection = PostgresAdapter.acquire_connection(self.profile) + self.assertEquals(connection.get('type'), 'postgres') + except ValidationException as e: + self.fail('got ValidationException: {}'.format(str(e))) + except BaseException as e: + self.fail('validation failed with unknown exception: {}' + .format(str(e))) + + def test_acquire_connection(self): + connection = PostgresAdapter.acquire_connection(self.profile) + + self.assertEquals(connection.get('state'), 'open') + self.assertNotEquals(connection.get('handle'), None) + + def test__get_connection(self): + connection = PostgresAdapter.get_connection(self.profile) + duplicate = PostgresAdapter.get_connection(self.profile) + + self.assertEquals(connection.get('state'), 'open') + self.assertNotEquals(connection.get('handle'), None) + + self.assertEquals(connection, duplicate) diff --git a/tox.ini b/tox.ini index 45e20a0a89b..41e45380735 100644 --- a/tox.ini +++ b/tox.ini @@ -1,47 +1,74 @@ [tox] -envlist = unit-py27, unit-py35, integration-py27, integration-py35, pep8 +skipsdist = True +envlist = unit-py27, unit-py36, integration-postgres-py27, integration-postgres-py36, integration-snowflake-py27, integration-snowflake-py36, pep8 [testenv:pep8] -basepython = python3.5 +basepython = python3.6 commands = /bin/bash -c '$(which pep8) dbt/ --exclude dbt/templates.py' deps = - -rrequirements.txt - -rdev_requirements.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt [testenv:unit-py27] basepython = python2.7 -commands = /bin/bash -c '$(which nosetests) -v test/unit' +commands = /bin/bash -c '$(which nosetests) -v {posargs} test/unit' deps = - -rrequirements.txt - -rdev_requirements.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt -[testenv:unit-py35] -basepython = python3.5 -commands = /bin/bash -c '$(which nosetests) -v test/unit' +[testenv:unit-py36] +basepython = python3.6 +commands = /bin/bash -c '{envpython} $(which nosetests) -v {posargs} test/unit' deps = - -rrequirements.txt - -rdev_requirements.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt -[testenv:integration-py27] +[testenv:integration-postgres-py27] basepython = python2.7 -commands = /bin/bash -c 'HOME=/root/ DBT_INVOCATION_ENV=ci-circle {envpython} $(which nosetests) -v --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' +passenv = * +setenv = + HOME=/root/ +commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=postgres {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -rrequirements.txt - -rdev_requirements.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt -[testenv:integration-py35] -basepython = python3.5 -commands = /bin/bash -c 'HOME=/root/ DBT_INVOCATION_ENV=ci-circle {envpython} $(which nosetests) -v --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' +[testenv:integration-snowflake-py27] +basepython = python2.7 +passenv = * +setenv = + HOME=/root/ +commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=snowflake {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -rrequirements.txt - -rdev_requirements.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt + +[testenv:integration-postgres-py36] +basepython = python3.6 +passenv = * +setenv = + HOME=/root/ +commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=postgres {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt + +[testenv:integration-snowflake-py36] +basepython = python3.6 +passenv = * +setenv = + HOME=/root/ +commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=snowflake {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev_requirements.txt [testenv:pywin] basepython = {env:PYTHON:}\python.exe setenv = DBT_CONFIG_DIR = ~/.dbt DBT_INVOCATION_ENV = ci-appveyor -commands = nosetests -v --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/unit test/integration/ +commands = nosetests -v -A "type!='snowflake'" --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/unit test/integration deps = -rrequirements.txt -rdev_requirements.txt