Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Singer Postgres --> Postgres replication demo #2

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*.idea*
*.iml
12 changes: 12 additions & 0 deletions psql2psql/first_time_setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
brew install pipx
pipx install virtualenv

#https://github.com/brianmario/mysql2/issues/795
export LDFLAGS="-L/usr/local/opt/openssl/lib"
export CPPFLAGS="-I/usr/local/opt/openssl/include"
./install_connector.sh tap-postgres
./install_connector target-postgres singer-target-postgres
sherifnada marked this conversation as resolved.
Show resolved Hide resolved



42 changes: 42 additions & 0 deletions psql2psql/init_db.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/bin/bash
CONTAINER_NAME=
IMG=postgres:11.6

function _delete_container_if_exists() {
# kill if exists.
echo "killing ${CONTAINER_NAME}"
docker rm -f ${CONTAINER_NAME} 2> /dev/null || true
}

function _wait_until_db_is_ready() {
RETRIES=5

until docker exec "${CONTAINER_NAME}" /bin/sh -c "psql -h localhost -p 5432 -U postgres -d postgres -c \"select 1\"" > /dev/null 2>&1 || [ $RETRIES -eq 0 ]; do
echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..."
sleep 1
done
}

function _seed_db(){
INIT_SCRIPT=$1
INIT_SCRIPT_TARGET_NAME=initscript
echo $INIT_SCRIPT_TARGET_NAME
docker cp "$INIT_SCRIPT" ${CONTAINER_NAME}:/docker-entrypoint-initdb.d/"${INIT_SCRIPT_TARGET_NAME}".sql
docker exec -u postgres "$CONTAINER_NAME" psql postgres postgres -f docker-entrypoint-initdb.d/"${INIT_SCRIPT_TARGET_NAME}".sql
}

function main(){
CONTAINER_NAME=$1
PORT=$2
INIT_SCRIPT=$3
_delete_container_if_exists
docker run --name "${CONTAINER_NAME}" -p "${2}":5432 -d ${IMG} # run db
_wait_until_db_is_ready

if [ -n "$INIT_SCRIPT" ]; then
_seed_db "$INIT_SCRIPT"
fi
}

main "$@"

10 changes: 10 additions & 0 deletions psql2psql/install_connector.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash
# Use python 3.7 to guarantee compatibility with deps

VENV=$1
PIP_PACKAGE_NAME=${2:-$VENV}

virtualenv -p python3.7 "$VENV"
source "${VENV}"/bin/activate
pip install "$PIP_PACKAGE_NAME"

7 changes: 7 additions & 0 deletions psql2psql/local_tap_config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"dbname":"postgres",
"user":"postgres",
"password":"",
"port":"9988",
"host":"localhost"
}
250 changes: 250 additions & 0 deletions psql2psql/populate_db.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,250 @@
CREATE TABLE countries (
id integer,
country_code VARCHAR(200),
name VARCHAR(200),
ios3 VARCHAR(200),
numcode VARCHAR(200),
withdrawn boolean
);

INSERT INTO countries (id, country_code, name, ios3, numcode, withdrawn)
VALUES
(1, 'AF', 'Afghanistan', 'AFG', 4, 'f'),
(2, 'AL', 'Albania', 'ALB', 8, 'f'),
(3, 'DZ', 'Algeria', 'DZA', 12, 'f'),
(4, 'AS', 'American Samoa', 'ASM', 16, 'f'),
(5, 'AD', 'Andorra', 'AND', 20, 'f'),
(6, 'AO', 'Angola', 'AGO', 24, 'f'),
(7, 'AI', 'Anguilla', 'AIA', 660, 'f'),
(8, 'AQ', 'Antarctica', NULL, NULL, 'f'),
(9, 'AG', 'Antigua and Barbuda', 'ATG', 28, 'f'),
(10, 'AR', 'Argentina', 'ARG', 32, 'f'),
(11, 'AM', 'Armenia', 'ARM', 51, 'f'),
(12, 'AW', 'Aruba', 'ABW', 533, 'f'),
(13, 'AU', 'Australia', 'AUS', 36, 'f'),
(14, 'AT', 'Austria', 'AUT', 40, 'f'),
(15, 'AZ', 'Azerbaijan', 'AZE', 31, 'f'),
(16, 'BS', 'Bahamas', 'BHS', 44, 'f'),
(17, 'BH', 'Bahrain', 'BHR', 48, 'f'),
(18, 'BD', 'Bangladesh', 'BGD', 50, 'f'),
(19, 'BB', 'Barbados', 'BRB', 52, 'f'),
(20, 'BY', 'Belarus', 'BLR', 112, 'f'),
(21, 'BE', 'Belgium', 'BEL', 56, 'f'),
(22, 'BZ', 'Belize', 'BLZ', 84, 'f'),
(23, 'BJ', 'Benin', 'BEN', 204, 'f'),
(24, 'BM', 'Bermuda', 'BMU', 60, 'f'),
(25, 'BT', 'Bhutan', 'BTN', 64, 'f'),
(26, 'BO', 'Bolivia', 'BOL', 68, 'f'),
(27, 'BA', 'Bosnia and Herzegovina', 'BIH', 70, 'f'),
(28, 'BW', 'Botswana', 'BWA', 72, 'f'),
(29, 'BV', 'Bouvet Island', NULL, NULL, 'f'),
(30, 'BR', 'Brazil', 'BRA', 76, 'f'),
(31, 'IO', 'British Indian Ocean Territory', NULL, NULL, 'f'),
(32, 'BN', 'Brunei Darussalam', 'BRN', 96, 'f'),
(33, 'BG', 'Bulgaria', 'BGR', 100, 'f'),
(34, 'BF', 'Burkina Faso', 'BFA', 854, 'f'),
(35, 'BI', 'Burundi', 'BDI', 108, 'f'),
(36, 'KH', 'Cambodia', 'KHM', 116, 'f'),
(37, 'CM', 'Cameroon', 'CMR', 120, 'f'),
(38, 'CA', 'Canada', 'CAN', 124, 'f'),
(39, 'CV', 'Cape Verde', 'CPV', 132, 'f'),
(40, 'KY', 'Cayman Islands', 'CYM', 136, 'f'),
(41, 'CF', 'Central African Republic', 'CAF', 140, 'f'),
(42, 'TD', 'Chad', 'TCD', 148, 'f'),
(43, 'CL', 'Chile', 'CHL', 152, 'f'),
(44, 'CN', 'China', 'CHN', 156, 'f'),
(45, 'CX', 'Christmas Island', NULL, NULL, 'f'),
(46, 'CC', 'Cocos (Keeling) Islands', NULL, NULL, 'f'),
(47, 'CO', 'Colombia', 'COL', 170, 'f'),
(48, 'KM', 'Comoros', 'COM', 174, 'f'),
(49, 'CG', 'Congo', 'COG', 178, 'f'),
(50, 'CD', 'Congo, the Democratic Republic of the', 'COD', 180, 'f'),
(51, 'CK', 'Cook Islands', 'COK', 184, 'f'),
(52, 'CR', 'Costa Rica', 'CRI', 188, 'f'),
(53, 'CI', 'Cote D''Ivoire', 'CIV', 384, 'f'),
(54, 'HR', 'Croatia', 'HRV', 191, 'f'),
(55, 'CU', 'Cuba', 'CUB', 192, 'f'),
(56, 'CY', 'Cyprus', 'CYP', 196, 'f'),
(57, 'CZ', 'Czech Republic', 'CZE', 203, 'f'),
(58, 'DK', 'Denmark', 'DNK', 208, 'f'),
(59, 'DJ', 'Djibouti', 'DJI', 262, 'f'),
(60, 'DM', 'Dominica', 'DMA', 212, 'f'),
(61, 'DO', 'Dominican Republic', 'DOM', 214, 'f'),
(62, 'EC', 'Ecuador', 'ECU', 218, 'f'),
(63, 'EG', 'Egypt', 'EGY', 818, 'f'),
(64, 'SV', 'El Salvador', 'SLV', 222, 'f'),
(65, 'GQ', 'Equatorial Guinea', 'GNQ', 226, 'f'),
(66, 'ER', 'Eritrea', 'ERI', 232, 'f'),
(67, 'EE', 'Estonia', 'EST', 233, 'f'),
(68, 'ET', 'Ethiopia', 'ETH', 231, 'f'),
(69, 'FK', 'Falkland Islands (Malvinas)', 'FLK', 238, 'f'),
(70, 'FO', 'Faroe Islands', 'FRO', 234, 'f'),
(71, 'FJ', 'Fiji', 'FJI', 242, 'f'),
(72, 'FI', 'Finland', 'FIN', 246, 'f'),
(73, 'FR', 'France', 'FRA', 250, 'f'),
(74, 'GF', 'French Guiana', 'GUF', 254, 'f'),
(75, 'PF', 'French Polynesia', 'PYF', 258, 'f'),
(76, 'TF', 'French Southern Territories', NULL, NULL, 'f'),
(77, 'GA', 'Gabon', 'GAB', 266, 'f'),
(78, 'GM', 'Gambia', 'GMB', 270, 'f'),
(79, 'GE', 'Georgia', 'GEO', 268, 'f'),
(80, 'DE', 'Germany', 'DEU', 276, 'f'),
(81, 'GH', 'Ghana', 'GHA', 288, 'f'),
(82, 'GI', 'Gibraltar', 'GIB', 292, 'f'),
(83, 'GR', 'Greece', 'GRC', 300, 'f'),
(84, 'GL', 'Greenland', 'GRL', 304, 'f'),
(85, 'GD', 'Grenada', 'GRD', 308, 'f'),
(86, 'GP', 'Guadeloupe', 'GLP', 312, 'f'),
(87, 'GU', 'Guam', 'GUM', 316, 'f'),
(88, 'GT', 'Guatemala', 'GTM', 320, 'f'),
(89, 'GN', 'Guinea', 'GIN', 324, 'f'),
(90, 'GW', 'Guinea-Bissau', 'GNB', 624, 'f'),
(91, 'GY', 'Guyana', 'GUY', 328, 'f'),
(92, 'HT', 'Haiti', 'HTI', 332, 'f'),
(93, 'HM', 'Heard Island and Mcdonald Islands', NULL, NULL, 'f'),
(94, 'VA', 'Vatican City State', 'VAT', 336, 'f'),
(95, 'HN', 'Honduras', 'HND', 340, 'f'),
(96, 'HK', 'Hong Kong', 'HKG', 344, 'f'),
(97, 'HU', 'Hungary', 'HUN', 348, 'f'),
(98, 'IS', 'Iceland', 'ISL', 352, 'f'),
(99, 'IN', 'India', 'IND', 356, 'f'),
(100, 'ID', 'Indonesia', 'IDN', 360, 'f'),
(101, 'IR', 'Iran', 'IRN', 364, 'f'),
(102, 'IQ', 'Iraq', 'IRQ', 368, 'f'),
(103, 'IE', 'Ireland', 'IRL', 372, 'f'),
(104, 'IL', 'Israel', 'ISR', 376, 'f'),
(105, 'IT', 'Italy', 'ITA', 380, 'f'),
(106, 'JM', 'Jamaica', 'JAM', 388, 'f'),
(107, 'JP', 'Japan', 'JPN', 392, 'f'),
(108, 'JO', 'Jordan', 'JOR', 400, 'f'),
(109, 'KZ', 'Kazakhstan', 'KAZ', 398, 'f'),
(110, 'KE', 'Kenya', 'KEN', 404, 'f'),
(111, 'KI', 'Kiribati', 'KIR', 296, 'f'),
(112, 'KP', 'North Korea', 'PRK', 408, 'f'),
(113, 'KR', 'South Korea', 'KOR', 410, 'f'),
(114, 'KW', 'Kuwait', 'KWT', 414, 'f'),
(115, 'KG', 'Kyrgyzstan', 'KGZ', 417, 'f'),
(116, 'LA', 'Lao People''s Democratic Republic', 'LAO', 418, 'f'),
(117, 'LV', 'Latvia', 'LVA', 428, 'f'),
(118, 'LB', 'Lebanon', 'LBN', 422, 'f'),
(119, 'LS', 'Lesotho', 'LSO', 426, 'f'),
(120, 'LR', 'Liberia', 'LBR', 430, 'f'),
(121, 'LY', 'Libyan Arab Jamahiriya', 'LBY', 434, 'f'),
(122, 'LI', 'Liechtenstein', 'LIE', 438, 'f'),
(123, 'LT', 'Lithuania', 'LTU', 440, 'f'),
(124, 'LU', 'Luxembourg', 'LUX', 442, 'f'),
(125, 'MO', 'Macao', 'MAC', 446, 'f'),
(126, 'MK', 'Macedonia', 'MKD', 807, 'f'),
(127, 'MG', 'Madagascar', 'MDG', 450, 'f'),
(128, 'MW', 'Malawi', 'MWI', 454, 'f'),
(129, 'MY', 'Malaysia', 'MYS', 458, 'f'),
(130, 'MV', 'Maldives', 'MDV', 462, 'f'),
(131, 'ML', 'Mali', 'MLI', 466, 'f'),
(132, 'MT', 'Malta', 'MLT', 470, 'f'),
(133, 'MH', 'Marshall Islands', 'MHL', 584, 'f'),
(134, 'MQ', 'Martinique', 'MTQ', 474, 'f'),
(135, 'MR', 'Mauritania', 'MRT', 478, 'f'),
(136, 'MU', 'Mauritius', 'MUS', 480, 'f'),
(137, 'YT', 'Mayotte', NULL, NULL, 'f'),
(138, 'MX', 'Mexico', 'MEX', 484, 'f'),
(139, 'FM', 'Micronesia, Federated States of', 'FSM', 583, 'f'),
(140, 'MD', 'Moldova, Republic of', 'MDA', 498, 'f'),
(141, 'MC', 'Monaco', 'MCO', 492, 'f'),
(142, 'MN', 'Mongolia', 'MNG', 496, 'f'),
(143, 'MS', 'Montserrat', 'MSR', 500, 'f'),
(144, 'MA', 'Morocco', 'MAR', 504, 'f'),
(145, 'MZ', 'Mozambique', 'MOZ', 508, 'f'),
(146, 'MM', 'Myanmar', 'MMR', 104, 'f'),
(147, 'NA', 'Namibia', 'NAM', 516, 'f'),
(148, 'NR', 'Nauru', 'NRU', 520, 'f'),
(149, 'NP', 'Nepal', 'NPL', 524, 'f'),
(150, 'NL', 'Netherlands', 'NLD', 528, 'f'),
(151, 'AN', 'Netherlands Antilles', 'ANT', 530, 'f'),
(152, 'NC', 'New Caledonia', 'NCL', 540, 'f'),
(153, 'NZ', 'New Zealand', 'NZL', 554, 'f'),
(154, 'NI', 'Nicaragua', 'NIC', 558, 'f'),
(155, 'NE', 'Niger', 'NER', 562, 'f'),
(156, 'NG', 'Nigeria', 'NGA', 566, 'f'),
(157, 'NU', 'Niue', 'NIU', 570, 'f'),
(158, 'NF', 'Norfolk Island', 'NFK', 574, 'f'),
(159, 'MP', 'Northern Mariana Islands', 'MNP', 580, 'f'),
(160, 'NO', 'Norway', 'NOR', 578, 'f'),
(161, 'OM', 'Oman', 'OMN', 512, 'f'),
(162, 'PK', 'Pakistan', 'PAK', 586, 'f'),
(163, 'PW', 'Palau', 'PLW', 585, 'f'),
(164, 'PS', 'Palestinian Territory, Occupied', NULL, NULL, 'f'),
(165, 'PA', 'Panama', 'PAN', 591, 'f'),
(166, 'PG', 'Papua New Guinea', 'PNG', 598, 'f'),
(167, 'PY', 'Paraguay', 'PRY', 600, 'f'),
(168, 'PE', 'Peru', 'PER', 604, 'f'),
(169, 'PH', 'Philippines', 'PHL', 608, 'f'),
(170, 'PN', 'Pitcairn', 'PCN', 612, 'f'),
(171, 'PL', 'Poland', 'POL', 616, 'f'),
(172, 'PT', 'Portugal', 'PRT', 620, 'f'),
(173, 'PR', 'Puerto Rico', 'PRI', 630, 'f'),
(174, 'QA', 'Qatar', 'QAT', 634, 'f'),
(175, 'RE', 'Reunion', 'REU', 638, 'f'),
(176, 'RO', 'Romania', 'ROM', 642, 'f'),
(177, 'RU', 'Russian Federation', 'RUS', 643, 'f'),
(178, 'RW', 'Rwanda', 'RWA', 646, 'f'),
(179, 'SH', 'Saint Helena', 'SHN', 654, 'f'),
(180, 'KN', 'Saint Kitts and Nevis', 'KNA', 659, 'f'),
(181, 'LC', 'Saint Lucia', 'LCA', 662, 'f'),
(182, 'PM', 'Saint Pierre and Miquelon', 'SPM', 666, 'f'),
(183, 'VC', 'Saint Vincent and the Grenadines', 'VCT', 670, 'f'),
(184, 'WS', 'Samoa', 'WSM', 882, 'f'),
(185, 'SM', 'San Marino', 'SMR', 674, 'f'),
(186, 'ST', 'Sao Tome and Principe', 'STP', 678, 'f'),
(187, 'SA', 'Saudi Arabia', 'SAU', 682, 'f'),
(188, 'SN', 'Senegal', 'SEN', 686, 'f'),
(189, 'CS', 'Serbia and Montenegro', NULL, NULL, 'f'),
(190, 'SC', 'Seychelles', 'SYC', 690, 'f'),
(191, 'SL', 'Sierra Leone', 'SLE', 694, 'f'),
(192, 'SG', 'Singapore', 'SGP', 702, 'f'),
(193, 'SK', 'Slovakia', 'SVK', 703, 'f'),
(194, 'SI', 'Slovenia', 'SVN', 705, 'f'),
(195, 'SB', 'Solomon Islands', 'SLB', 90, 'f'),
(196, 'SO', 'Somalia', 'SOM', 706, 'f'),
(197, 'ZA', 'South Africa', 'ZAF', 710, 'f'),
(198, 'GS', 'South Georgia and the South Sandwich Islands', NULL, NULL, 'f'),
(199, 'ES', 'Spain', 'ESP', 724, 'f'),
(200, 'LK', 'Sri Lanka', 'LKA', 144, 'f'),
(201, 'SD', 'Sudan', 'SDN', 736, 'f'),
(202, 'SR', 'Suriname', 'SUR', 740, 'f'),
(203, 'SJ', 'Svalbard and Jan Mayen', 'SJM', 744, 'f'),
(204, 'SZ', 'Swaziland', 'SWZ', 748, 'f'),
(205, 'SE', 'Sweden', 'SWE', 752, 'f'),
(206, 'CH', 'Switzerland', 'CHE', 756, 'f'),
(207, 'SY', 'Syrian Arab Republic', 'SYR', 760, 'f'),
(208, 'TW', 'Taiwan, Province of China', 'TWN', 158, 'f'),
(209, 'TJ', 'Tajikistan', 'TJK', 762, 'f'),
(210, 'TZ', 'Tanzania, United Republic of', 'TZA', 834, 'f'),
(211, 'TH', 'Thailand', 'THA', 764, 'f'),
(212, 'TL', 'Timor-Leste', NULL, NULL, 'f'),
(213, 'TG', 'Togo', 'TGO', 768, 'f'),
(214, 'TK', 'Tokelau', 'TKL', 772, 'f'),
(215, 'TO', 'Tonga', 'TON', 776, 'f'),
(216, 'TT', 'Trinidad and Tobago', 'TTO', 780, 'f'),
(217, 'TN', 'Tunisia', 'TUN', 788, 'f'),
(218, 'TR', 'Turkey', 'TUR', 792, 'f'),
(219, 'TM', 'Turkmenistan', 'TKM', 795, 'f'),
(220, 'TC', 'Turks and Caicos Islands', 'TCA', 796, 'f'),
(221, 'TV', 'Tuvalu', 'TUV', 798, 'f'),
(222, 'UG', 'Uganda', 'UGA', 800, 'f'),
(223, 'UA', 'Ukraine', 'UKR', 804, 'f'),
(224, 'AE', 'United Arab Emirates', 'ARE', 784, 'f'),
(225, 'GB', 'United Kingdom', 'GBR', 826, 'f'),
(226, 'US', 'United States', 'USA', 840, 'f'),
(227, 'UM', 'United States Minor Outlying Islands', NULL, NULL, 'f'),
(228, 'UY', 'Uruguay', 'URY', 858, 'f'),
(229, 'UZ', 'Uzbekistan', 'UZB', 860, 'f'),
(230, 'VU', 'Vanuatu', 'VUT', 548, 'f'),
(231, 'VE', 'Venezuela', 'VEN', 862, 'f'),
(232, 'VN', 'Viet Nam', 'VNM', 704, 'f'),
(233, 'VG', 'Virgin Islands, British', 'VGB', 92, 'f'),
(234, 'VI', 'Virgin Islands, U.S.', 'VIR', 850, 'f'),
(235, 'WF', 'Wallis and Futuna', 'WLF', 876, 'f'),
(236, 'EH', 'Western Sahara', 'ESH', 732, 'f'),
(237, 'YE', 'Yemen', 'YEM', 887, 'f'),
(238, 'ZM', 'Zambia', 'ZMB', 894, 'f'),
(239, 'ZW', 'Zimbabwe', 'ZWE', 716, 'f');
7 changes: 7 additions & 0 deletions psql2psql/remote_tap_config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"dbname":"pfmegrnargs",
"user":"reader",
"password":"NWDMCE5xdipIjRrp",
"port":"5432",
"host":"hh-pgsql-public.ebi.ac.uk"
}
18 changes: 18 additions & 0 deletions psql2psql/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/bin/sh

# Setup local source database
./init_db.sh src-postgres 9988 ./populate_db.sql

# Setup Target database
./init_db.sh target-postgres 9999

# Load data from local database

# load catalog then edit it
#./tap-postgres/bin/tap-postgres --config local_tap_config.json --discover > local_tap_catalog.json
./tap-postgres/bin/tap-postgres --config local_tap_config.json --properties local_tap_catalog.json | ./singer-target-postgres/bin/target-postgres --config target_config.json >> state.json
sherifnada marked this conversation as resolved.
Show resolved Hide resolved

# Load data from remote database
#./tap-postgres/bin/tap-postgres --config remote_tap_config.json
./tap-postgres/bin/tap-postgres --config remote_tap_config.json --properties remote_tap_catalog.json | ./singer-target-postgres/bin/target-postgres --config target_config.json >> state.json
sherifnada marked this conversation as resolved.
Show resolved Hide resolved

10 changes: 10 additions & 0 deletions psql2psql/target_config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"postgres_host": "localhost",
"postgres_port": 9999,
"postgres_database": "postgres",
"postgres_username": "postgres",
"postgres_password": "",
"postgres_schema": "public",
"persist_empty_tables": true
}