Singer Postgres --> Postgres replication demo

.gitignore

@@ -0,0 +1,2 @@
+*.idea*
+*.iml

psql2psql/first_time_setup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+brew install pipx
+pipx install virtualenv
+
+#https://github.com/brianmario/mysql2/issues/795
+export LDFLAGS="-L/usr/local/opt/openssl/lib"
+export CPPFLAGS="-I/usr/local/opt/openssl/include"
+./install_connector.sh tap-postgres
+./install_connector target-postgres singer-target-postgres
+
+
+

psql2psql/init_db.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+CONTAINER_NAME=
+IMG=postgres:11.6
+
+function _delete_container_if_exists() {
+  # kill if exists.
+  echo "killing ${CONTAINER_NAME}"
+  docker rm -f ${CONTAINER_NAME} 2> /dev/null || true
+}
+
+function _wait_until_db_is_ready() {
+  RETRIES=5
+
+  until docker exec "${CONTAINER_NAME}" /bin/sh -c "psql -h localhost -p 5432 -U postgres -d postgres -c \"select 1\"" > /dev/null 2>&1 || [ $RETRIES -eq 0 ]; do
+    echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..."
+    sleep 1
+  done
+}
+
+function _seed_db(){
+  INIT_SCRIPT=$1
+  INIT_SCRIPT_TARGET_NAME=initscript
+  echo $INIT_SCRIPT_TARGET_NAME
+  docker cp "$INIT_SCRIPT" ${CONTAINER_NAME}:/docker-entrypoint-initdb.d/"${INIT_SCRIPT_TARGET_NAME}".sql
+  docker exec -u postgres "$CONTAINER_NAME" psql postgres postgres -f docker-entrypoint-initdb.d/"${INIT_SCRIPT_TARGET_NAME}".sql
+}
+
+function main(){
+  CONTAINER_NAME=$1
+  PORT=$2
+  INIT_SCRIPT=$3
+  _delete_container_if_exists
+  docker run --name "${CONTAINER_NAME}" -p "${2}":5432 -d ${IMG} # run db
+  _wait_until_db_is_ready
+
+  if [ -n "$INIT_SCRIPT" ]; then
+    _seed_db "$INIT_SCRIPT"
+  fi
+}
+
+main "$@"
+

psql2psql/install_connector.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Use python 3.7 to guarantee compatibility with deps
+
+VENV=$1
+PIP_PACKAGE_NAME=${2:-$VENV}
+
+virtualenv -p python3.7 "$VENV"
+source "${VENV}"/bin/activate
+pip install "$PIP_PACKAGE_NAME"
+

psql2psql/local_tap_config.json

@@ -0,0 +1,7 @@
+{
+  "dbname":"postgres",
+  "user":"postgres",
+  "password":"",
+  "port":"9988",
+  "host":"localhost"
+}

psql2psql/populate_db.sql

@@ -0,0 +1,250 @@
+CREATE TABLE countries (
+   id   integer,
+   country_code VARCHAR(200),
+   name VARCHAR(200),
+   ios3 VARCHAR(200),
+   numcode VARCHAR(200),
+   withdrawn boolean
+);
+
+INSERT INTO countries (id, country_code, name, ios3, numcode, withdrawn)
+VALUES
+(1, 'AF', 'Afghanistan', 'AFG', 4, 'f'),
+(2, 'AL', 'Albania', 'ALB', 8, 'f'),
+(3, 'DZ', 'Algeria', 'DZA', 12, 'f'),
+(4, 'AS', 'American Samoa', 'ASM', 16, 'f'),
+(5, 'AD', 'Andorra', 'AND', 20, 'f'),
+(6, 'AO', 'Angola', 'AGO', 24, 'f'),
+(7, 'AI', 'Anguilla', 'AIA', 660, 'f'),
+(8, 'AQ', 'Antarctica', NULL, NULL, 'f'),
+(9, 'AG', 'Antigua and Barbuda', 'ATG', 28, 'f'),
+(10, 'AR', 'Argentina', 'ARG', 32, 'f'),
+(11, 'AM', 'Armenia', 'ARM', 51, 'f'),
+(12, 'AW', 'Aruba', 'ABW', 533, 'f'),
+(13, 'AU', 'Australia', 'AUS', 36, 'f'),
+(14, 'AT', 'Austria', 'AUT', 40, 'f'),
+(15, 'AZ', 'Azerbaijan', 'AZE', 31, 'f'),
+(16, 'BS', 'Bahamas', 'BHS', 44, 'f'),
+(17, 'BH', 'Bahrain', 'BHR', 48, 'f'),
+(18, 'BD', 'Bangladesh', 'BGD', 50, 'f'),
+(19, 'BB', 'Barbados', 'BRB', 52, 'f'),
+(20, 'BY', 'Belarus', 'BLR', 112, 'f'),
+(21, 'BE', 'Belgium', 'BEL', 56, 'f'),
+(22, 'BZ', 'Belize', 'BLZ', 84, 'f'),
+(23, 'BJ', 'Benin', 'BEN', 204, 'f'),
+(24, 'BM', 'Bermuda', 'BMU', 60, 'f'),
+(25, 'BT', 'Bhutan', 'BTN', 64, 'f'),
+(26, 'BO', 'Bolivia', 'BOL', 68, 'f'),
+(27, 'BA', 'Bosnia and Herzegovina', 'BIH', 70, 'f'),
+(28, 'BW', 'Botswana', 'BWA', 72, 'f'),
+(29, 'BV', 'Bouvet Island', NULL, NULL, 'f'),
+(30, 'BR', 'Brazil', 'BRA', 76, 'f'),
+(31, 'IO', 'British Indian Ocean Territory', NULL, NULL, 'f'),
+(32, 'BN', 'Brunei Darussalam', 'BRN', 96, 'f'),
+(33, 'BG', 'Bulgaria', 'BGR', 100, 'f'),
+(34, 'BF', 'Burkina Faso', 'BFA', 854, 'f'),
+(35, 'BI', 'Burundi', 'BDI', 108, 'f'),
+(36, 'KH', 'Cambodia', 'KHM', 116, 'f'),
+(37, 'CM', 'Cameroon', 'CMR', 120, 'f'),
+(38, 'CA', 'Canada', 'CAN', 124, 'f'),
+(39, 'CV', 'Cape Verde', 'CPV', 132, 'f'),
+(40, 'KY', 'Cayman Islands', 'CYM', 136, 'f'),
+(41, 'CF', 'Central African Republic', 'CAF', 140, 'f'),
+(42, 'TD', 'Chad', 'TCD', 148, 'f'),
+(43, 'CL', 'Chile', 'CHL', 152, 'f'),
+(44, 'CN', 'China', 'CHN', 156, 'f'),
+(45, 'CX', 'Christmas Island', NULL, NULL, 'f'),
+(46, 'CC', 'Cocos (Keeling) Islands', NULL, NULL, 'f'),
+(47, 'CO', 'Colombia', 'COL', 170, 'f'),
+(48, 'KM', 'Comoros', 'COM', 174, 'f'),
+(49, 'CG', 'Congo', 'COG', 178, 'f'),
+(50, 'CD', 'Congo, the Democratic Republic of the', 'COD', 180, 'f'),
+(51, 'CK', 'Cook Islands', 'COK', 184, 'f'),
+(52, 'CR', 'Costa Rica', 'CRI', 188, 'f'),
+(53, 'CI', 'Cote D''Ivoire', 'CIV', 384, 'f'),
+(54, 'HR', 'Croatia', 'HRV', 191, 'f'),
+(55, 'CU', 'Cuba', 'CUB', 192, 'f'),
+(56, 'CY', 'Cyprus', 'CYP', 196, 'f'),
+(57, 'CZ', 'Czech Republic', 'CZE', 203, 'f'),
+(58, 'DK', 'Denmark', 'DNK', 208, 'f'),
+(59, 'DJ', 'Djibouti', 'DJI', 262, 'f'),
+(60, 'DM', 'Dominica', 'DMA', 212, 'f'),
+(61, 'DO', 'Dominican Republic', 'DOM', 214, 'f'),
+(62, 'EC', 'Ecuador', 'ECU', 218, 'f'),
+(63, 'EG', 'Egypt', 'EGY', 818, 'f'),
+(64, 'SV', 'El Salvador', 'SLV', 222, 'f'),
+(65, 'GQ', 'Equatorial Guinea', 'GNQ', 226, 'f'),
+(66, 'ER', 'Eritrea', 'ERI', 232, 'f'),
+(67, 'EE', 'Estonia', 'EST', 233, 'f'),
+(68, 'ET', 'Ethiopia', 'ETH', 231, 'f'),
+(69, 'FK', 'Falkland Islands (Malvinas)', 'FLK', 238, 'f'),
+(70, 'FO', 'Faroe Islands', 'FRO', 234, 'f'),
+(71, 'FJ', 'Fiji', 'FJI', 242, 'f'),
+(72, 'FI', 'Finland', 'FIN', 246, 'f'),
+(73, 'FR', 'France', 'FRA', 250, 'f'),
+(74, 'GF', 'French Guiana', 'GUF', 254, 'f'),
+(75, 'PF', 'French Polynesia', 'PYF', 258, 'f'),
+(76, 'TF', 'French Southern Territories', NULL, NULL, 'f'),
+(77, 'GA', 'Gabon', 'GAB', 266, 'f'),
+(78, 'GM', 'Gambia', 'GMB', 270, 'f'),
+(79, 'GE', 'Georgia', 'GEO', 268, 'f'),
+(80, 'DE', 'Germany', 'DEU', 276, 'f'),
+(81, 'GH', 'Ghana', 'GHA', 288, 'f'),
+(82, 'GI', 'Gibraltar', 'GIB', 292, 'f'),
+(83, 'GR', 'Greece', 'GRC', 300, 'f'),
+(84, 'GL', 'Greenland', 'GRL', 304, 'f'),
+(85, 'GD', 'Grenada', 'GRD', 308, 'f'),
+(86, 'GP', 'Guadeloupe', 'GLP', 312, 'f'),
+(87, 'GU', 'Guam', 'GUM', 316, 'f'),
+(88, 'GT', 'Guatemala', 'GTM', 320, 'f'),
+(89, 'GN', 'Guinea', 'GIN', 324, 'f'),
+(90, 'GW', 'Guinea-Bissau', 'GNB', 624, 'f'),
+(91, 'GY', 'Guyana', 'GUY', 328, 'f'),
+(92, 'HT', 'Haiti', 'HTI', 332, 'f'),
+(93, 'HM', 'Heard Island and Mcdonald Islands', NULL, NULL, 'f'),
+(94, 'VA', 'Vatican City State', 'VAT', 336, 'f'),
+(95, 'HN', 'Honduras', 'HND', 340, 'f'),
+(96, 'HK', 'Hong Kong', 'HKG', 344, 'f'),
+(97, 'HU', 'Hungary', 'HUN', 348, 'f'),
+(98, 'IS', 'Iceland', 'ISL', 352, 'f'),
+(99, 'IN', 'India', 'IND', 356, 'f'),
+(100, 'ID', 'Indonesia', 'IDN', 360, 'f'),
+(101, 'IR', 'Iran', 'IRN', 364, 'f'),
+(102, 'IQ', 'Iraq', 'IRQ', 368, 'f'),
+(103, 'IE', 'Ireland', 'IRL', 372, 'f'),
+(104, 'IL', 'Israel', 'ISR', 376, 'f'),
+(105, 'IT', 'Italy', 'ITA', 380, 'f'),
+(106, 'JM', 'Jamaica', 'JAM', 388, 'f'),
+(107, 'JP', 'Japan', 'JPN', 392, 'f'),
+(108, 'JO', 'Jordan', 'JOR', 400, 'f'),
+(109, 'KZ', 'Kazakhstan', 'KAZ', 398, 'f'),
+(110, 'KE', 'Kenya', 'KEN', 404, 'f'),
+(111, 'KI', 'Kiribati', 'KIR', 296, 'f'),
+(112, 'KP', 'North Korea', 'PRK', 408, 'f'),
+(113, 'KR', 'South Korea', 'KOR', 410, 'f'),
+(114, 'KW', 'Kuwait', 'KWT', 414, 'f'),
+(115, 'KG', 'Kyrgyzstan', 'KGZ', 417, 'f'),
+(116, 'LA', 'Lao People''s Democratic Republic', 'LAO', 418, 'f'),
+(117, 'LV', 'Latvia', 'LVA', 428, 'f'),
+(118, 'LB', 'Lebanon', 'LBN', 422, 'f'),
+(119, 'LS', 'Lesotho', 'LSO', 426, 'f'),
+(120, 'LR', 'Liberia', 'LBR', 430, 'f'),
+(121, 'LY', 'Libyan Arab Jamahiriya', 'LBY', 434, 'f'),
+(122, 'LI', 'Liechtenstein', 'LIE', 438, 'f'),
+(123, 'LT', 'Lithuania', 'LTU', 440, 'f'),
+(124, 'LU', 'Luxembourg', 'LUX', 442, 'f'),
+(125, 'MO', 'Macao', 'MAC', 446, 'f'),
+(126, 'MK', 'Macedonia', 'MKD', 807, 'f'),
+(127, 'MG', 'Madagascar', 'MDG', 450, 'f'),
+(128, 'MW', 'Malawi', 'MWI', 454, 'f'),
+(129, 'MY', 'Malaysia', 'MYS', 458, 'f'),
+(130, 'MV', 'Maldives', 'MDV', 462, 'f'),
+(131, 'ML', 'Mali', 'MLI', 466, 'f'),
+(132, 'MT', 'Malta', 'MLT', 470, 'f'),
+(133, 'MH', 'Marshall Islands', 'MHL', 584, 'f'),
+(134, 'MQ', 'Martinique', 'MTQ', 474, 'f'),
+(135, 'MR', 'Mauritania', 'MRT', 478, 'f'),
+(136, 'MU', 'Mauritius', 'MUS', 480, 'f'),
+(137, 'YT', 'Mayotte', NULL, NULL, 'f'),
+(138, 'MX', 'Mexico', 'MEX', 484, 'f'),
+(139, 'FM', 'Micronesia, Federated States of', 'FSM', 583, 'f'),
+(140, 'MD', 'Moldova, Republic of', 'MDA', 498, 'f'),
+(141, 'MC', 'Monaco', 'MCO', 492, 'f'),
+(142, 'MN', 'Mongolia', 'MNG', 496, 'f'),
+(143, 'MS', 'Montserrat', 'MSR', 500, 'f'),
+(144, 'MA', 'Morocco', 'MAR', 504, 'f'),
+(145, 'MZ', 'Mozambique', 'MOZ', 508, 'f'),
+(146, 'MM', 'Myanmar', 'MMR', 104, 'f'),
+(147, 'NA', 'Namibia', 'NAM', 516, 'f'),
+(148, 'NR', 'Nauru', 'NRU', 520, 'f'),
+(149, 'NP', 'Nepal', 'NPL', 524, 'f'),
+(150, 'NL', 'Netherlands', 'NLD', 528, 'f'),
+(151, 'AN', 'Netherlands Antilles', 'ANT', 530, 'f'),
+(152, 'NC', 'New Caledonia', 'NCL', 540, 'f'),
+(153, 'NZ', 'New Zealand', 'NZL', 554, 'f'),
+(154, 'NI', 'Nicaragua', 'NIC', 558, 'f'),
+(155, 'NE', 'Niger', 'NER', 562, 'f'),
+(156, 'NG', 'Nigeria', 'NGA', 566, 'f'),
+(157, 'NU', 'Niue', 'NIU', 570, 'f'),
+(158, 'NF', 'Norfolk Island', 'NFK', 574, 'f'),
+(159, 'MP', 'Northern Mariana Islands', 'MNP', 580, 'f'),
+(160, 'NO', 'Norway', 'NOR', 578, 'f'),
+(161, 'OM', 'Oman', 'OMN', 512, 'f'),
+(162, 'PK', 'Pakistan', 'PAK', 586, 'f'),
+(163, 'PW', 'Palau', 'PLW', 585, 'f'),
+(164, 'PS', 'Palestinian Territory, Occupied', NULL, NULL, 'f'),
+(165, 'PA', 'Panama', 'PAN', 591, 'f'),
+(166, 'PG', 'Papua New Guinea', 'PNG', 598, 'f'),
+(167, 'PY', 'Paraguay', 'PRY', 600, 'f'),
+(168, 'PE', 'Peru', 'PER', 604, 'f'),
+(169, 'PH', 'Philippines', 'PHL', 608, 'f'),
+(170, 'PN', 'Pitcairn', 'PCN', 612, 'f'),
+(171, 'PL', 'Poland', 'POL', 616, 'f'),
+(172, 'PT', 'Portugal', 'PRT', 620, 'f'),
+(173, 'PR', 'Puerto Rico', 'PRI', 630, 'f'),
+(174, 'QA', 'Qatar', 'QAT', 634, 'f'),
+(175, 'RE', 'Reunion', 'REU', 638, 'f'),
+(176, 'RO', 'Romania', 'ROM', 642, 'f'),
+(177, 'RU', 'Russian Federation', 'RUS', 643, 'f'),
+(178, 'RW', 'Rwanda', 'RWA', 646, 'f'),
+(179, 'SH', 'Saint Helena', 'SHN', 654, 'f'),
+(180, 'KN', 'Saint Kitts and Nevis', 'KNA', 659, 'f'),
+(181, 'LC', 'Saint Lucia', 'LCA', 662, 'f'),
+(182, 'PM', 'Saint Pierre and Miquelon', 'SPM', 666, 'f'),
+(183, 'VC', 'Saint Vincent and the Grenadines', 'VCT', 670, 'f'),
+(184, 'WS', 'Samoa', 'WSM', 882, 'f'),
+(185, 'SM', 'San Marino', 'SMR', 674, 'f'),
+(186, 'ST', 'Sao Tome and Principe', 'STP', 678, 'f'),
+(187, 'SA', 'Saudi Arabia', 'SAU', 682, 'f'),
+(188, 'SN', 'Senegal', 'SEN', 686, 'f'),
+(189, 'CS', 'Serbia and Montenegro', NULL, NULL, 'f'),
+(190, 'SC', 'Seychelles', 'SYC', 690, 'f'),
+(191, 'SL', 'Sierra Leone', 'SLE', 694, 'f'),
+(192, 'SG', 'Singapore', 'SGP', 702, 'f'),
+(193, 'SK', 'Slovakia', 'SVK', 703, 'f'),
+(194, 'SI', 'Slovenia', 'SVN', 705, 'f'),
+(195, 'SB', 'Solomon Islands', 'SLB', 90, 'f'),
+(196, 'SO', 'Somalia', 'SOM', 706, 'f'),
+(197, 'ZA', 'South Africa', 'ZAF', 710, 'f'),
+(198, 'GS', 'South Georgia and the South Sandwich Islands', NULL, NULL, 'f'),
+(199, 'ES', 'Spain', 'ESP', 724, 'f'),
+(200, 'LK', 'Sri Lanka', 'LKA', 144, 'f'),
+(201, 'SD', 'Sudan', 'SDN', 736, 'f'),
+(202, 'SR', 'Suriname', 'SUR', 740, 'f'),
+(203, 'SJ', 'Svalbard and Jan Mayen', 'SJM', 744, 'f'),
+(204, 'SZ', 'Swaziland', 'SWZ', 748, 'f'),
+(205, 'SE', 'Sweden', 'SWE', 752, 'f'),
+(206, 'CH', 'Switzerland', 'CHE', 756, 'f'),
+(207, 'SY', 'Syrian Arab Republic', 'SYR', 760, 'f'),
+(208, 'TW', 'Taiwan, Province of China', 'TWN', 158, 'f'),
+(209, 'TJ', 'Tajikistan', 'TJK', 762, 'f'),
+(210, 'TZ', 'Tanzania, United Republic of', 'TZA', 834, 'f'),
+(211, 'TH', 'Thailand', 'THA', 764, 'f'),
+(212, 'TL', 'Timor-Leste', NULL, NULL, 'f'),
+(213, 'TG', 'Togo', 'TGO', 768, 'f'),
+(214, 'TK', 'Tokelau', 'TKL', 772, 'f'),
+(215, 'TO', 'Tonga', 'TON', 776, 'f'),
+(216, 'TT', 'Trinidad and Tobago', 'TTO', 780, 'f'),
+(217, 'TN', 'Tunisia', 'TUN', 788, 'f'),
+(218, 'TR', 'Turkey', 'TUR', 792, 'f'),
+(219, 'TM', 'Turkmenistan', 'TKM', 795, 'f'),
+(220, 'TC', 'Turks and Caicos Islands', 'TCA', 796, 'f'),
+(221, 'TV', 'Tuvalu', 'TUV', 798, 'f'),
+(222, 'UG', 'Uganda', 'UGA', 800, 'f'),
+(223, 'UA', 'Ukraine', 'UKR', 804, 'f'),
+(224, 'AE', 'United Arab Emirates', 'ARE', 784, 'f'),
+(225, 'GB', 'United Kingdom', 'GBR', 826, 'f'),
+(226, 'US', 'United States', 'USA', 840, 'f'),
+(227, 'UM', 'United States Minor Outlying Islands', NULL, NULL, 'f'),
+(228, 'UY', 'Uruguay', 'URY', 858, 'f'),
+(229, 'UZ', 'Uzbekistan', 'UZB', 860, 'f'),
+(230, 'VU', 'Vanuatu', 'VUT', 548, 'f'),
+(231, 'VE', 'Venezuela', 'VEN', 862, 'f'),
+(232, 'VN', 'Viet Nam', 'VNM', 704, 'f'),
+(233, 'VG', 'Virgin Islands, British', 'VGB', 92, 'f'),
+(234, 'VI', 'Virgin Islands, U.S.', 'VIR', 850, 'f'),
+(235, 'WF', 'Wallis and Futuna', 'WLF', 876, 'f'),
+(236, 'EH', 'Western Sahara', 'ESH', 732, 'f'),
+(237, 'YE', 'Yemen', 'YEM', 887, 'f'),
+(238, 'ZM', 'Zambia', 'ZMB', 894, 'f'),
+(239, 'ZW', 'Zimbabwe', 'ZWE', 716, 'f');

psql2psql/remote_tap_config.json

@@ -0,0 +1,7 @@
+{
+  "dbname":"pfmegrnargs",
+  "user":"reader",
+  "password":"NWDMCE5xdipIjRrp",
+  "port":"5432",
+  "host":"hh-pgsql-public.ebi.ac.uk"
+}

psql2psql/run.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Setup local source database
+./init_db.sh src-postgres 9988 ./populate_db.sql
+
+# Setup Target database
+./init_db.sh target-postgres 9999
+
+# Load data from local database
+
+# load catalog then edit it
+#./tap-postgres/bin/tap-postgres --config local_tap_config.json --discover > local_tap_catalog.json
+./tap-postgres/bin/tap-postgres --config local_tap_config.json --properties local_tap_catalog.json | ./singer-target-postgres/bin/target-postgres --config target_config.json >> state.json
+
+# Load data from remote database
+#./tap-postgres/bin/tap-postgres --config remote_tap_config.json
+./tap-postgres/bin/tap-postgres --config remote_tap_config.json --properties remote_tap_catalog.json | ./singer-target-postgres/bin/target-postgres --config target_config.json >> state.json
+

psql2psql/target_config.json

@@ -0,0 +1,10 @@
+{
+  "postgres_host": "localhost",
+  "postgres_port": 9999,
+  "postgres_database": "postgres",
+  "postgres_username": "postgres",
+  "postgres_password": "",
+  "postgres_schema": "public",
+  "persist_empty_tables": true
+}
+