Skip to content

Commit

Permalink
add root primary identifiers to meta
Browse files Browse the repository at this point in the history
  • Loading branch information
@toluaina
toluaina committed Aug 25, 2024
1 parent 7cdd542 commit e665625
Show file tree
Hide file tree
Showing 8 changed files with 406 additions and 191 deletions.
12 changes: 12 additions & 0 deletions pgsync/sync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -983,6 +983,18 @@ def sync(

row[META] = Transform.get_primary_keys(keys)

if node.is_root:
primary_key_values: t.List[str] = list(
map(str, primary_keys)
)
primary_key_names: t.List[str] = [
primary_key.name for primary_key in node.primary_keys
]
# TODO: add support for composite pkeys
row[META][node.table] = {
primary_key_names[0]: [primary_key_values[0]],
}

if self.verbose:
print(f"{(i+1)})")
print(f"pkeys: {primary_keys}")
Expand Down
13 changes: 6 additions & 7 deletions requirements/base.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
#
async-timeout==4.0.3
# via redis
boto3==1.34.159
boto3==1.35.5
# via -r requirements/base.in
botocore==1.34.159
botocore==1.35.5
# via
# boto3
# s3transfer
Expand All @@ -27,25 +27,25 @@ elasticsearch==8.15.0
# via
# -r requirements/base.in
# elasticsearch-dsl
elasticsearch-dsl==8.15.0
elasticsearch-dsl==8.15.1
# via -r requirements/base.in
environs==11.0.0
# via -r requirements/base.in
events==0.5
# via opensearch-py
greenlet==3.0.3
# via sqlalchemy
idna==3.7
idna==3.8
# via requests
jmespath==1.0.1
# via
# boto3
# botocore
marshmallow==3.21.3
marshmallow==3.22.0
# via environs
opensearch-dsl==2.1.0
# via -r requirements/base.in
opensearch-py==2.6.0
opensearch-py==2.7.1
# via opensearch-dsl
packaging==24.1
# via marshmallow
Expand Down Expand Up @@ -74,7 +74,6 @@ s3transfer==0.10.2
six==1.16.0
# via
# opensearch-dsl
# opensearch-py
# python-dateutil
sqlalchemy==2.0.32
# via -r requirements/base.in
Expand Down
15 changes: 7 additions & 8 deletions requirements/dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ async-timeout==4.0.3
# via redis
black==24.8.0
# via -r requirements/dev.in
boto3==1.34.159
boto3==1.35.5
# via -r requirements/base.in
botocore==1.34.159
botocore==1.35.5
# via
# boto3
# s3transfer
Expand Down Expand Up @@ -39,15 +39,15 @@ elasticsearch==8.15.0
# via
# -r requirements/base.in
# elasticsearch-dsl
elasticsearch-dsl==8.15.0
elasticsearch-dsl==8.15.1
# via -r requirements/base.in
environs==11.0.0
# via -r requirements/base.in
events==0.5
# via opensearch-py
exceptiongroup==1.2.2
# via pytest
faker==27.0.0
faker==28.0.0
# via -r requirements/dev.in
filelock==3.15.4
# via virtualenv
Expand All @@ -59,7 +59,7 @@ greenlet==3.0.3
# via sqlalchemy
identify==2.6.0
# via pre-commit
idna==3.7
idna==3.8
# via requests
iniconfig==2.0.0
# via pytest
Expand All @@ -69,7 +69,7 @@ jmespath==1.0.1
# via
# boto3
# botocore
marshmallow==3.21.3
marshmallow==3.22.0
# via environs
mccabe==0.7.0
# via flake8
Expand All @@ -81,7 +81,7 @@ nodeenv==1.9.1
# via pre-commit
opensearch-dsl==2.1.0
# via -r requirements/base.in
opensearch-py==2.6.0
opensearch-py==2.7.1
# via opensearch-dsl
packaging==24.1
# via
Expand Down Expand Up @@ -140,7 +140,6 @@ s3transfer==0.10.2
six==1.16.0
# via
# opensearch-dsl
# opensearch-py
# python-dateutil
sqlalchemy==2.0.32
# via -r requirements/base.in
Expand Down
28 changes: 23 additions & 5 deletions tests/test_sync_nested_children.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,7 @@ def test_sync(self, sync, nodes, data):
"_index": "testdb",
"_source": {
"_meta": {
"book": {"isbn": ["abc"]},
"author": {"id": [1, 4]},
"book_author": {"id": [1, 4]},
"book_language": {"id": [1, 4, 7, 9]},
Expand Down Expand Up @@ -449,6 +450,7 @@ def test_sync(self, sync, nodes, data):
"_index": "testdb",
"_source": {
"_meta": {
"book": {"isbn": ["def"]},
"author": {"id": [1, 2]},
"book_author": {"id": [2, 5]},
"book_language": {"id": [2, 5, 8]},
Expand Down Expand Up @@ -506,6 +508,7 @@ def test_sync(self, sync, nodes, data):
"_index": "testdb",
"_source": {
"_meta": {
"book": {"isbn": ["ghi"]},
"author": {"id": [2, 3]},
"book_author": {"id": [3, 6]},
"book_language": {"id": [3, 6]},
Expand Down Expand Up @@ -708,6 +711,7 @@ def test_insert_root(
"_index": "testdb",
"_source": {
"_meta": {
"book": {"isbn": ["jkl"]},
"author": {"id": [5]},
"book_author": {"id": [7]},
"book_language": {"id": [10, 11]},
Expand Down Expand Up @@ -753,6 +757,7 @@ def test_insert_root(
"_index": "testdb",
"_source": {
"_meta": {
"book": {"isbn": ["mno"]},
"author": {"id": [5]},
"book_author": {"id": [8]},
"book_language": {"id": [12, 13]},
Expand Down Expand Up @@ -834,6 +839,7 @@ def test_update_root(self, data, nodes, book_cls):
"_index": "testdb",
"_source": {
"_meta": {
"book": {"isbn": ["abc"]},
"author": {"id": [1, 4]},
"book_author": {"id": [1, 4]},
"book_language": {"id": [1, 4, 7, 9]},
Expand Down Expand Up @@ -1001,6 +1007,7 @@ def poll_db():
expected = [
{
"_meta": {
"book": {"isbn": ["def"]},
"author": {"id": [1, 2]},
"book_author": {"id": [2, 5]},
"book_language": {"id": [2, 5, 8]},
Expand Down Expand Up @@ -1054,6 +1061,7 @@ def poll_db():
},
{
"_meta": {
"book": {"isbn": ["ghi"]},
"author": {"id": [2, 3]},
"book_author": {"id": [3, 6]},
"book_language": {"id": [3, 6]},
Expand Down Expand Up @@ -1174,7 +1182,7 @@ def test_insert_through_child_op2(
"isbn": "abc",
"group": None,
"title": "The Tiger Club",
"_meta": {},
"_meta": {"book": {"isbn": ["abc"]}},
},
},
{
Expand All @@ -1184,7 +1192,7 @@ def test_insert_through_child_op2(
"isbn": "def",
"group": None,
"title": "The Lion Club",
"_meta": {},
"_meta": {"book": {"isbn": ["def"]}},
},
},
{
Expand All @@ -1194,7 +1202,7 @@ def test_insert_through_child_op2(
"isbn": "ghi",
"group": None,
"title": "The Rabbit Club",
"_meta": {},
"_meta": {"book": {"isbn": ["ghi"]}},
},
},
]
Expand Down Expand Up @@ -1249,6 +1257,7 @@ def poll_db():
],
"title": "The Tiger Club",
"_meta": {
"book": {"isbn": ["abc"]},
"group": {"id": [1, 2]},
"book_group": {"id": [1, 2]},
},
Expand All @@ -1261,7 +1270,7 @@ def poll_db():
"isbn": "def",
"group": None,
"title": "The Lion Club",
"_meta": {},
"_meta": {"book": {"isbn": ["def"]}},
},
},
{
Expand All @@ -1271,7 +1280,7 @@ def poll_db():
"isbn": "ghi",
"group": None,
"title": "The Rabbit Club",
"_meta": {},
"_meta": {"book": {"isbn": ["ghi"]}},
},
},
]
Expand Down Expand Up @@ -1346,6 +1355,7 @@ def test_insert_through_child_op(
expected = [
{
"_meta": {
"book": {"isbn": ["abc"]},
"author": {"id": [1, 4, 5]},
"book_author": {"id": [1, 4, 7]},
"book_language": {"id": [1, 4, 7, 9]},
Expand Down Expand Up @@ -1413,6 +1423,7 @@ def test_insert_through_child_op(
},
{
"_meta": {
"book": {"isbn": ["def"]},
"author": {"id": [1, 2]},
"book_author": {"id": [2, 5]},
"book_language": {"id": [2, 5, 8]},
Expand Down Expand Up @@ -1466,6 +1477,7 @@ def test_insert_through_child_op(
},
{
"_meta": {
"book": {"isbn": ["ghi"]},
"author": {"id": [2, 3]},
"book_author": {"id": [3, 6]},
"book_language": {"id": [3, 6]},
Expand Down Expand Up @@ -1602,6 +1614,7 @@ def test_update_through_child_op(
expected = [
{
"_meta": {
"book": {"isbn": ["abc"]},
"author": {"id": [4, 5]},
"book_author": {"id": [1, 4]},
"book_language": {"id": [1, 4, 7, 9]},
Expand Down Expand Up @@ -1656,6 +1669,7 @@ def test_update_through_child_op(
},
{
"_meta": {
"book": {"isbn": ["def"]},
"author": {"id": [1, 2]},
"book_author": {"id": [2, 5]},
"book_language": {"id": [2, 5, 8]},
Expand Down Expand Up @@ -1709,6 +1723,7 @@ def test_update_through_child_op(
},
{
"_meta": {
"book": {"isbn": ["ghi"]},
"author": {"id": [2, 3]},
"book_author": {"id": [3, 6]},
"book_language": {"id": [3, 6]},
Expand Down Expand Up @@ -1818,6 +1833,7 @@ def test_delete_through_child_op(self, sync, data, nodes, book_author_cls):
expected = [
{
"_meta": {
"book": {"isbn": ["abc"]},
"book_language": {"id": [1, 4, 7, 9]},
"book_subject": {"id": [1, 4, 6]},
"language": {"id": [1, 2, 3, 4]},
Expand All @@ -1840,6 +1856,7 @@ def test_delete_through_child_op(self, sync, data, nodes, book_author_cls):
},
{
"_meta": {
"book": {"isbn": ["def"]},
"author": {"id": [1, 2]},
"book_author": {"id": [2, 5]},
"book_language": {"id": [2, 5, 8]},
Expand Down Expand Up @@ -1893,6 +1910,7 @@ def test_delete_through_child_op(self, sync, data, nodes, book_author_cls):
},
{
"_meta": {
"book": {"isbn": ["ghi"]},
"author": {"id": [2, 3]},
"book_author": {"id": [3, 6]},
"book_language": {"id": [3, 6]},
Expand Down
Loading

0 comments on commit e665625

Please sign in to comment.