diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..14f0e69d8c19 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.idea* +*.iml diff --git a/psql2psql/first_time_setup.sh b/psql2psql/first_time_setup.sh new file mode 100755 index 000000000000..f139446c66e7 --- /dev/null +++ b/psql2psql/first_time_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +brew install pipx +pipx install virtualenv + +#https://github.com/brianmario/mysql2/issues/795 +export LDFLAGS="-L/usr/local/opt/openssl/lib" +export CPPFLAGS="-I/usr/local/opt/openssl/include" +./install_connector.sh tap-postgres +./install_connector.sh target-postgres singer-target-postgres + + diff --git a/psql2psql/init_db.sh b/psql2psql/init_db.sh new file mode 100755 index 000000000000..824f85e1a273 --- /dev/null +++ b/psql2psql/init_db.sh @@ -0,0 +1,42 @@ +#!/bin/bash +CONTAINER_NAME= +IMG=postgres:11.6 + +function _delete_container_if_exists() { + # kill if exists. + echo "killing ${CONTAINER_NAME}" + docker rm -f ${CONTAINER_NAME} 2> /dev/null || true +} + +function _wait_until_db_is_ready() { + RETRIES=5 + + until docker exec "${CONTAINER_NAME}" /bin/sh -c "psql -h localhost -p 5432 -U postgres -d postgres -c \"select 1\"" > /dev/null 2>&1 || [ $RETRIES -eq 0 ]; do + echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..." + sleep 1 + done +} + +function _seed_db(){ + INIT_SCRIPT=$1 + INIT_SCRIPT_TARGET_NAME=initscript + echo $INIT_SCRIPT_TARGET_NAME + docker cp "$INIT_SCRIPT" ${CONTAINER_NAME}:/docker-entrypoint-initdb.d/"${INIT_SCRIPT_TARGET_NAME}".sql + docker exec -u postgres "$CONTAINER_NAME" psql postgres postgres -f docker-entrypoint-initdb.d/"${INIT_SCRIPT_TARGET_NAME}".sql +} + +function main(){ + CONTAINER_NAME=$1 + PORT=$2 + INIT_SCRIPT=$3 + _delete_container_if_exists + docker run --name "${CONTAINER_NAME}" -p "${2}":5432 -d ${IMG} # run db + _wait_until_db_is_ready + + if [ -n "$INIT_SCRIPT" ]; then + _seed_db "$INIT_SCRIPT" + fi +} + +main "$@" + diff --git a/psql2psql/install_connector.sh b/psql2psql/install_connector.sh new file mode 100755 index 000000000000..43dbe2cf2190 --- /dev/null +++ b/psql2psql/install_connector.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Use python 3.7 to guarantee compatibility with deps + +VENV=$1 +PIP_PACKAGE_NAME=${2:-$VENV} + +virtualenv -p python3.7 "$VENV" +source "${VENV}"/bin/activate +pip install "$PIP_PACKAGE_NAME" + diff --git a/psql2psql/local_tap_config.json b/psql2psql/local_tap_config.json new file mode 100644 index 000000000000..145a79fcc443 --- /dev/null +++ b/psql2psql/local_tap_config.json @@ -0,0 +1,7 @@ +{ + "dbname":"postgres", + "user":"postgres", + "password":"", + "port":"9988", + "host":"localhost" +} diff --git a/psql2psql/populate_db.sql b/psql2psql/populate_db.sql new file mode 100644 index 000000000000..c47d29141b7b --- /dev/null +++ b/psql2psql/populate_db.sql @@ -0,0 +1,250 @@ +CREATE TABLE countries ( + id integer, + country_code VARCHAR(200), + name VARCHAR(200), + ios3 VARCHAR(200), + numcode VARCHAR(200), + withdrawn boolean +); + +INSERT INTO countries (id, country_code, name, ios3, numcode, withdrawn) +VALUES +(1, 'AF', 'Afghanistan', 'AFG', 4, 'f'), +(2, 'AL', 'Albania', 'ALB', 8, 'f'), +(3, 'DZ', 'Algeria', 'DZA', 12, 'f'), +(4, 'AS', 'American Samoa', 'ASM', 16, 'f'), +(5, 'AD', 'Andorra', 'AND', 20, 'f'), +(6, 'AO', 'Angola', 'AGO', 24, 'f'), +(7, 'AI', 'Anguilla', 'AIA', 660, 'f'), +(8, 'AQ', 'Antarctica', NULL, NULL, 'f'), +(9, 'AG', 'Antigua and Barbuda', 'ATG', 28, 'f'), +(10, 'AR', 'Argentina', 'ARG', 32, 'f'), +(11, 'AM', 'Armenia', 'ARM', 51, 'f'), +(12, 'AW', 'Aruba', 'ABW', 533, 'f'), +(13, 'AU', 'Australia', 'AUS', 36, 'f'), +(14, 'AT', 'Austria', 'AUT', 40, 'f'), +(15, 'AZ', 'Azerbaijan', 'AZE', 31, 'f'), +(16, 'BS', 'Bahamas', 'BHS', 44, 'f'), +(17, 'BH', 'Bahrain', 'BHR', 48, 'f'), +(18, 'BD', 'Bangladesh', 'BGD', 50, 'f'), +(19, 'BB', 'Barbados', 'BRB', 52, 'f'), +(20, 'BY', 'Belarus', 'BLR', 112, 'f'), +(21, 'BE', 'Belgium', 'BEL', 56, 'f'), +(22, 'BZ', 'Belize', 'BLZ', 84, 'f'), +(23, 'BJ', 'Benin', 'BEN', 204, 'f'), +(24, 'BM', 'Bermuda', 'BMU', 60, 'f'), +(25, 'BT', 'Bhutan', 'BTN', 64, 'f'), +(26, 'BO', 'Bolivia', 'BOL', 68, 'f'), +(27, 'BA', 'Bosnia and Herzegovina', 'BIH', 70, 'f'), +(28, 'BW', 'Botswana', 'BWA', 72, 'f'), +(29, 'BV', 'Bouvet Island', NULL, NULL, 'f'), +(30, 'BR', 'Brazil', 'BRA', 76, 'f'), +(31, 'IO', 'British Indian Ocean Territory', NULL, NULL, 'f'), +(32, 'BN', 'Brunei Darussalam', 'BRN', 96, 'f'), +(33, 'BG', 'Bulgaria', 'BGR', 100, 'f'), +(34, 'BF', 'Burkina Faso', 'BFA', 854, 'f'), +(35, 'BI', 'Burundi', 'BDI', 108, 'f'), +(36, 'KH', 'Cambodia', 'KHM', 116, 'f'), +(37, 'CM', 'Cameroon', 'CMR', 120, 'f'), +(38, 'CA', 'Canada', 'CAN', 124, 'f'), +(39, 'CV', 'Cape Verde', 'CPV', 132, 'f'), +(40, 'KY', 'Cayman Islands', 'CYM', 136, 'f'), +(41, 'CF', 'Central African Republic', 'CAF', 140, 'f'), +(42, 'TD', 'Chad', 'TCD', 148, 'f'), +(43, 'CL', 'Chile', 'CHL', 152, 'f'), +(44, 'CN', 'China', 'CHN', 156, 'f'), +(45, 'CX', 'Christmas Island', NULL, NULL, 'f'), +(46, 'CC', 'Cocos (Keeling) Islands', NULL, NULL, 'f'), +(47, 'CO', 'Colombia', 'COL', 170, 'f'), +(48, 'KM', 'Comoros', 'COM', 174, 'f'), +(49, 'CG', 'Congo', 'COG', 178, 'f'), +(50, 'CD', 'Congo, the Democratic Republic of the', 'COD', 180, 'f'), +(51, 'CK', 'Cook Islands', 'COK', 184, 'f'), +(52, 'CR', 'Costa Rica', 'CRI', 188, 'f'), +(53, 'CI', 'Cote D''Ivoire', 'CIV', 384, 'f'), +(54, 'HR', 'Croatia', 'HRV', 191, 'f'), +(55, 'CU', 'Cuba', 'CUB', 192, 'f'), +(56, 'CY', 'Cyprus', 'CYP', 196, 'f'), +(57, 'CZ', 'Czech Republic', 'CZE', 203, 'f'), +(58, 'DK', 'Denmark', 'DNK', 208, 'f'), +(59, 'DJ', 'Djibouti', 'DJI', 262, 'f'), +(60, 'DM', 'Dominica', 'DMA', 212, 'f'), +(61, 'DO', 'Dominican Republic', 'DOM', 214, 'f'), +(62, 'EC', 'Ecuador', 'ECU', 218, 'f'), +(63, 'EG', 'Egypt', 'EGY', 818, 'f'), +(64, 'SV', 'El Salvador', 'SLV', 222, 'f'), +(65, 'GQ', 'Equatorial Guinea', 'GNQ', 226, 'f'), +(66, 'ER', 'Eritrea', 'ERI', 232, 'f'), +(67, 'EE', 'Estonia', 'EST', 233, 'f'), +(68, 'ET', 'Ethiopia', 'ETH', 231, 'f'), +(69, 'FK', 'Falkland Islands (Malvinas)', 'FLK', 238, 'f'), +(70, 'FO', 'Faroe Islands', 'FRO', 234, 'f'), +(71, 'FJ', 'Fiji', 'FJI', 242, 'f'), +(72, 'FI', 'Finland', 'FIN', 246, 'f'), +(73, 'FR', 'France', 'FRA', 250, 'f'), +(74, 'GF', 'French Guiana', 'GUF', 254, 'f'), +(75, 'PF', 'French Polynesia', 'PYF', 258, 'f'), +(76, 'TF', 'French Southern Territories', NULL, NULL, 'f'), +(77, 'GA', 'Gabon', 'GAB', 266, 'f'), +(78, 'GM', 'Gambia', 'GMB', 270, 'f'), +(79, 'GE', 'Georgia', 'GEO', 268, 'f'), +(80, 'DE', 'Germany', 'DEU', 276, 'f'), +(81, 'GH', 'Ghana', 'GHA', 288, 'f'), +(82, 'GI', 'Gibraltar', 'GIB', 292, 'f'), +(83, 'GR', 'Greece', 'GRC', 300, 'f'), +(84, 'GL', 'Greenland', 'GRL', 304, 'f'), +(85, 'GD', 'Grenada', 'GRD', 308, 'f'), +(86, 'GP', 'Guadeloupe', 'GLP', 312, 'f'), +(87, 'GU', 'Guam', 'GUM', 316, 'f'), +(88, 'GT', 'Guatemala', 'GTM', 320, 'f'), +(89, 'GN', 'Guinea', 'GIN', 324, 'f'), +(90, 'GW', 'Guinea-Bissau', 'GNB', 624, 'f'), +(91, 'GY', 'Guyana', 'GUY', 328, 'f'), +(92, 'HT', 'Haiti', 'HTI', 332, 'f'), +(93, 'HM', 'Heard Island and Mcdonald Islands', NULL, NULL, 'f'), +(94, 'VA', 'Vatican City State', 'VAT', 336, 'f'), +(95, 'HN', 'Honduras', 'HND', 340, 'f'), +(96, 'HK', 'Hong Kong', 'HKG', 344, 'f'), +(97, 'HU', 'Hungary', 'HUN', 348, 'f'), +(98, 'IS', 'Iceland', 'ISL', 352, 'f'), +(99, 'IN', 'India', 'IND', 356, 'f'), +(100, 'ID', 'Indonesia', 'IDN', 360, 'f'), +(101, 'IR', 'Iran', 'IRN', 364, 'f'), +(102, 'IQ', 'Iraq', 'IRQ', 368, 'f'), +(103, 'IE', 'Ireland', 'IRL', 372, 'f'), +(104, 'IL', 'Israel', 'ISR', 376, 'f'), +(105, 'IT', 'Italy', 'ITA', 380, 'f'), +(106, 'JM', 'Jamaica', 'JAM', 388, 'f'), +(107, 'JP', 'Japan', 'JPN', 392, 'f'), +(108, 'JO', 'Jordan', 'JOR', 400, 'f'), +(109, 'KZ', 'Kazakhstan', 'KAZ', 398, 'f'), +(110, 'KE', 'Kenya', 'KEN', 404, 'f'), +(111, 'KI', 'Kiribati', 'KIR', 296, 'f'), +(112, 'KP', 'North Korea', 'PRK', 408, 'f'), +(113, 'KR', 'South Korea', 'KOR', 410, 'f'), +(114, 'KW', 'Kuwait', 'KWT', 414, 'f'), +(115, 'KG', 'Kyrgyzstan', 'KGZ', 417, 'f'), +(116, 'LA', 'Lao People''s Democratic Republic', 'LAO', 418, 'f'), +(117, 'LV', 'Latvia', 'LVA', 428, 'f'), +(118, 'LB', 'Lebanon', 'LBN', 422, 'f'), +(119, 'LS', 'Lesotho', 'LSO', 426, 'f'), +(120, 'LR', 'Liberia', 'LBR', 430, 'f'), +(121, 'LY', 'Libyan Arab Jamahiriya', 'LBY', 434, 'f'), +(122, 'LI', 'Liechtenstein', 'LIE', 438, 'f'), +(123, 'LT', 'Lithuania', 'LTU', 440, 'f'), +(124, 'LU', 'Luxembourg', 'LUX', 442, 'f'), +(125, 'MO', 'Macao', 'MAC', 446, 'f'), +(126, 'MK', 'Macedonia', 'MKD', 807, 'f'), +(127, 'MG', 'Madagascar', 'MDG', 450, 'f'), +(128, 'MW', 'Malawi', 'MWI', 454, 'f'), +(129, 'MY', 'Malaysia', 'MYS', 458, 'f'), +(130, 'MV', 'Maldives', 'MDV', 462, 'f'), +(131, 'ML', 'Mali', 'MLI', 466, 'f'), +(132, 'MT', 'Malta', 'MLT', 470, 'f'), +(133, 'MH', 'Marshall Islands', 'MHL', 584, 'f'), +(134, 'MQ', 'Martinique', 'MTQ', 474, 'f'), +(135, 'MR', 'Mauritania', 'MRT', 478, 'f'), +(136, 'MU', 'Mauritius', 'MUS', 480, 'f'), +(137, 'YT', 'Mayotte', NULL, NULL, 'f'), +(138, 'MX', 'Mexico', 'MEX', 484, 'f'), +(139, 'FM', 'Micronesia, Federated States of', 'FSM', 583, 'f'), +(140, 'MD', 'Moldova, Republic of', 'MDA', 498, 'f'), +(141, 'MC', 'Monaco', 'MCO', 492, 'f'), +(142, 'MN', 'Mongolia', 'MNG', 496, 'f'), +(143, 'MS', 'Montserrat', 'MSR', 500, 'f'), +(144, 'MA', 'Morocco', 'MAR', 504, 'f'), +(145, 'MZ', 'Mozambique', 'MOZ', 508, 'f'), +(146, 'MM', 'Myanmar', 'MMR', 104, 'f'), +(147, 'NA', 'Namibia', 'NAM', 516, 'f'), +(148, 'NR', 'Nauru', 'NRU', 520, 'f'), +(149, 'NP', 'Nepal', 'NPL', 524, 'f'), +(150, 'NL', 'Netherlands', 'NLD', 528, 'f'), +(151, 'AN', 'Netherlands Antilles', 'ANT', 530, 'f'), +(152, 'NC', 'New Caledonia', 'NCL', 540, 'f'), +(153, 'NZ', 'New Zealand', 'NZL', 554, 'f'), +(154, 'NI', 'Nicaragua', 'NIC', 558, 'f'), +(155, 'NE', 'Niger', 'NER', 562, 'f'), +(156, 'NG', 'Nigeria', 'NGA', 566, 'f'), +(157, 'NU', 'Niue', 'NIU', 570, 'f'), +(158, 'NF', 'Norfolk Island', 'NFK', 574, 'f'), +(159, 'MP', 'Northern Mariana Islands', 'MNP', 580, 'f'), +(160, 'NO', 'Norway', 'NOR', 578, 'f'), +(161, 'OM', 'Oman', 'OMN', 512, 'f'), +(162, 'PK', 'Pakistan', 'PAK', 586, 'f'), +(163, 'PW', 'Palau', 'PLW', 585, 'f'), +(164, 'PS', 'Palestinian Territory, Occupied', NULL, NULL, 'f'), +(165, 'PA', 'Panama', 'PAN', 591, 'f'), +(166, 'PG', 'Papua New Guinea', 'PNG', 598, 'f'), +(167, 'PY', 'Paraguay', 'PRY', 600, 'f'), +(168, 'PE', 'Peru', 'PER', 604, 'f'), +(169, 'PH', 'Philippines', 'PHL', 608, 'f'), +(170, 'PN', 'Pitcairn', 'PCN', 612, 'f'), +(171, 'PL', 'Poland', 'POL', 616, 'f'), +(172, 'PT', 'Portugal', 'PRT', 620, 'f'), +(173, 'PR', 'Puerto Rico', 'PRI', 630, 'f'), +(174, 'QA', 'Qatar', 'QAT', 634, 'f'), +(175, 'RE', 'Reunion', 'REU', 638, 'f'), +(176, 'RO', 'Romania', 'ROM', 642, 'f'), +(177, 'RU', 'Russian Federation', 'RUS', 643, 'f'), +(178, 'RW', 'Rwanda', 'RWA', 646, 'f'), +(179, 'SH', 'Saint Helena', 'SHN', 654, 'f'), +(180, 'KN', 'Saint Kitts and Nevis', 'KNA', 659, 'f'), +(181, 'LC', 'Saint Lucia', 'LCA', 662, 'f'), +(182, 'PM', 'Saint Pierre and Miquelon', 'SPM', 666, 'f'), +(183, 'VC', 'Saint Vincent and the Grenadines', 'VCT', 670, 'f'), +(184, 'WS', 'Samoa', 'WSM', 882, 'f'), +(185, 'SM', 'San Marino', 'SMR', 674, 'f'), +(186, 'ST', 'Sao Tome and Principe', 'STP', 678, 'f'), +(187, 'SA', 'Saudi Arabia', 'SAU', 682, 'f'), +(188, 'SN', 'Senegal', 'SEN', 686, 'f'), +(189, 'CS', 'Serbia and Montenegro', NULL, NULL, 'f'), +(190, 'SC', 'Seychelles', 'SYC', 690, 'f'), +(191, 'SL', 'Sierra Leone', 'SLE', 694, 'f'), +(192, 'SG', 'Singapore', 'SGP', 702, 'f'), +(193, 'SK', 'Slovakia', 'SVK', 703, 'f'), +(194, 'SI', 'Slovenia', 'SVN', 705, 'f'), +(195, 'SB', 'Solomon Islands', 'SLB', 90, 'f'), +(196, 'SO', 'Somalia', 'SOM', 706, 'f'), +(197, 'ZA', 'South Africa', 'ZAF', 710, 'f'), +(198, 'GS', 'South Georgia and the South Sandwich Islands', NULL, NULL, 'f'), +(199, 'ES', 'Spain', 'ESP', 724, 'f'), +(200, 'LK', 'Sri Lanka', 'LKA', 144, 'f'), +(201, 'SD', 'Sudan', 'SDN', 736, 'f'), +(202, 'SR', 'Suriname', 'SUR', 740, 'f'), +(203, 'SJ', 'Svalbard and Jan Mayen', 'SJM', 744, 'f'), +(204, 'SZ', 'Swaziland', 'SWZ', 748, 'f'), +(205, 'SE', 'Sweden', 'SWE', 752, 'f'), +(206, 'CH', 'Switzerland', 'CHE', 756, 'f'), +(207, 'SY', 'Syrian Arab Republic', 'SYR', 760, 'f'), +(208, 'TW', 'Taiwan, Province of China', 'TWN', 158, 'f'), +(209, 'TJ', 'Tajikistan', 'TJK', 762, 'f'), +(210, 'TZ', 'Tanzania, United Republic of', 'TZA', 834, 'f'), +(211, 'TH', 'Thailand', 'THA', 764, 'f'), +(212, 'TL', 'Timor-Leste', NULL, NULL, 'f'), +(213, 'TG', 'Togo', 'TGO', 768, 'f'), +(214, 'TK', 'Tokelau', 'TKL', 772, 'f'), +(215, 'TO', 'Tonga', 'TON', 776, 'f'), +(216, 'TT', 'Trinidad and Tobago', 'TTO', 780, 'f'), +(217, 'TN', 'Tunisia', 'TUN', 788, 'f'), +(218, 'TR', 'Turkey', 'TUR', 792, 'f'), +(219, 'TM', 'Turkmenistan', 'TKM', 795, 'f'), +(220, 'TC', 'Turks and Caicos Islands', 'TCA', 796, 'f'), +(221, 'TV', 'Tuvalu', 'TUV', 798, 'f'), +(222, 'UG', 'Uganda', 'UGA', 800, 'f'), +(223, 'UA', 'Ukraine', 'UKR', 804, 'f'), +(224, 'AE', 'United Arab Emirates', 'ARE', 784, 'f'), +(225, 'GB', 'United Kingdom', 'GBR', 826, 'f'), +(226, 'US', 'United States', 'USA', 840, 'f'), +(227, 'UM', 'United States Minor Outlying Islands', NULL, NULL, 'f'), +(228, 'UY', 'Uruguay', 'URY', 858, 'f'), +(229, 'UZ', 'Uzbekistan', 'UZB', 860, 'f'), +(230, 'VU', 'Vanuatu', 'VUT', 548, 'f'), +(231, 'VE', 'Venezuela', 'VEN', 862, 'f'), +(232, 'VN', 'Viet Nam', 'VNM', 704, 'f'), +(233, 'VG', 'Virgin Islands, British', 'VGB', 92, 'f'), +(234, 'VI', 'Virgin Islands, U.S.', 'VIR', 850, 'f'), +(235, 'WF', 'Wallis and Futuna', 'WLF', 876, 'f'), +(236, 'EH', 'Western Sahara', 'ESH', 732, 'f'), +(237, 'YE', 'Yemen', 'YEM', 887, 'f'), +(238, 'ZM', 'Zambia', 'ZMB', 894, 'f'), +(239, 'ZW', 'Zimbabwe', 'ZWE', 716, 'f'); \ No newline at end of file diff --git a/psql2psql/remote_tap_config.json b/psql2psql/remote_tap_config.json new file mode 100644 index 000000000000..0ce819c3f8f2 --- /dev/null +++ b/psql2psql/remote_tap_config.json @@ -0,0 +1,7 @@ +{ + "dbname":"pfmegrnargs", + "user":"reader", + "password":"NWDMCE5xdipIjRrp", + "port":"5432", + "host":"hh-pgsql-public.ebi.ac.uk" +} diff --git a/psql2psql/run.sh b/psql2psql/run.sh new file mode 100755 index 000000000000..a34e7e71c086 --- /dev/null +++ b/psql2psql/run.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Setup local source database +./init_db.sh src-postgres 9988 ./populate_db.sql + +# Setup Target database +./init_db.sh target-postgres 9999 + +# Load data from local database + +# load catalog then edit it +#./tap-postgres/bin/tap-postgres --config local_tap_config.json --discover > local_tap_catalog.json +./tap-postgres/bin/tap-postgres --config local_tap_config.json --properties local_tap_catalog.json | ./target-postgres/bin/target-postgres --config target_config.json >> state.json + +# Load data from remote database +#./tap-postgres/bin/tap-postgres --config remote_tap_config.json +./tap-postgres/bin/tap-postgres --config remote_tap_config.json --properties remote_tap_catalog.json | ./target-postgres/bin/target-postgres --config target_config.json >> state.json diff --git a/psql2psql/target_config.json b/psql2psql/target_config.json new file mode 100644 index 000000000000..f303bef67ccf --- /dev/null +++ b/psql2psql/target_config.json @@ -0,0 +1,10 @@ +{ + "postgres_host": "localhost", + "postgres_port": 9999, + "postgres_database": "postgres", + "postgres_username": "postgres", + "postgres_password": "", + "postgres_schema": "public", + "persist_empty_tables": true +} +