From 0499f8d28d88e9de53cea30ef1645d7c230280b6 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Tue, 3 Aug 2021 17:55:28 +0300
Subject: [PATCH 01/14] Support migration originations

---
 src/dipdup/datasources/tzkt/datasource.py | 90 ++++++++++++++++++++++-
 src/dipdup/exceptions.py                  | 22 +++++-
 src/dipdup/index.py                       | 11 ++-
 src/dipdup/models.py                      |  1 +
 4 files changed, 118 insertions(+), 6 deletions(-)

diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py
index f1ac09abb..9d97bc2ef 100644
--- a/src/dipdup/datasources/tzkt/datasource.py
+++ b/src/dipdup/datasources/tzkt/datasource.py
@@ -1,10 +1,13 @@
 import asyncio
 import logging
+from collections import defaultdict
 from datetime import datetime, timezone
 from decimal import Decimal
 from enum import Enum
-from typing import Any, AsyncGenerator, Dict, List, NoReturn, Optional, Set, Tuple, cast
+from types import MethodType
+from typing import Any, AsyncGenerator, DefaultDict, Dict, List, NoReturn, Optional, Set, Tuple, cast
 
+from aiohttp import ClientResponseError
 from aiosignalrcore.hub.base_hub_connection import BaseHubConnection  # type: ignore
 from aiosignalrcore.hub_connection_builder import HubConnectionBuilder  # type: ignore
 from aiosignalrcore.messages.completion_message import CompletionMessage  # type: ignore
@@ -20,8 +23,9 @@
 )
 from dipdup.datasources.datasource import IndexDatasource, SubscriptionManager
 from dipdup.datasources.tzkt.enums import TzktMessageType
+from dipdup.exceptions import MissingOriginationError
 from dipdup.models import BigMapAction, BigMapData, BlockData, HeadBlockData, OperationData
-from dipdup.utils import split_by_chunks
+from dipdup.utils import groupby, split_by_chunks
 
 OperationID = int
 
@@ -43,6 +47,15 @@
     "hasInternals",
     "diffs",
 )
+ORIGINATION_MIGRATION_FIELDS = (
+    "id",
+    "level",
+    "timestamp",
+    "storage",
+    "diffs",
+    "account",
+    "balanceChange",
+)
 ORIGINATION_OPERATION_FIELDS = (
     *OPERATION_FIELDS,
     "originatedContract",
@@ -83,6 +96,7 @@ def __init__(
         transaction_addresses: Set[str],
         origination_addresses: Set[str],
         cache: bool = False,
+        migration_originations: List[OperationData] = None,
     ) -> None:
         self._datasource = datasource
         self._first_level = first_level
@@ -96,7 +110,12 @@ def __init__(
         self._heads: Dict[OperationFetcherChannel, int] = {}
         self._offsets: Dict[OperationFetcherChannel, int] = {}
         self._fetched: Dict[OperationFetcherChannel, bool] = {}
-        self._operations: Dict[int, List[OperationData]] = {}
+
+        self._operations: DefaultDict[int, List[OperationData]]
+        if migration_originations:
+            self._operations = groupby(migration_originations, lambda op: op.level)
+        else:
+            self._operations = defaultdict(list)
 
     def _get_operations_head(self, operations: List[OperationData]) -> int:
         """Get latest block level (head) of sorted operations batch"""
@@ -378,6 +397,32 @@ async def get_block(self, level: int) -> BlockData:
         )
         return self.convert_block(block_json)
 
+    async def get_migration_originations(self) -> List[OperationData]:
+        """Get contracts originated from migrations"""
+        self._logger.info('Fetching contracts originated with migrations')
+        # NOTE: Empty unwrapped request to ensure API supports migration originations
+        try:
+            await self._http._request(
+                'get',
+                url='v1/operations/migrations',
+                params={
+                    'kind': 'origination',
+                    'limit': 0,
+                },
+            )
+        except ClientResponseError:
+            return []
+
+        raw_migrations = await self._http.request(
+            'get',
+            url='v1/operations/migrations',
+            params={
+                'kind': 'origination',
+                'select': ','.join(ORIGINATION_MIGRATION_FIELDS),
+            },
+        )
+        return [self.convert_migration_origination(m) for m in raw_migrations]
+
     async def get_originations(
         self, addresses: Set[str], offset: int, first_level: int, last_level: int, cache: bool = False
     ) -> List[OperationData]:
@@ -719,6 +764,45 @@ def convert_operation(cls, operation_json: Dict[str, Any]) -> OperationData:
             diffs=operation_json.get('diffs'),
         )
 
+    @classmethod
+    def convert_migration_origination(cls, migration_origination_json: Dict[str, Any]) -> OperationData:
+        """Convert raw migration message from REST into dataclass"""
+        storage = migration_origination_json.get('storage')
+        # FIXME: Plain storage, has issues in codegen: KT1CpeSQKdkhWi4pinYcseCFKmDhs5M74BkU
+        if not isinstance(storage, Dict):
+            storage = {}
+
+        missing_fields = dict(
+            hash='',
+            counter=0,
+            sender_address='',
+            target_address=None,
+            initiator_address=None,
+        )
+        fake_operation_data = OperationData(
+            type='origination',
+            id=migration_origination_json['id'],
+            level=migration_origination_json['level'],
+            timestamp=cls._parse_timestamp(migration_origination_json['timestamp']),
+            block=migration_origination_json.get('block'),
+            originated_contract_address=migration_origination_json['account']['address'],
+            originated_contract_alias=migration_origination_json['account'].get('alias'),
+            amount=migration_origination_json['balanceChange'],
+            storage=storage,
+            diffs=migration_origination_json.get('diffs'),
+            status='applied',
+            has_internals=False,
+            **missing_fields,  # type: ignore
+        )
+
+        def __getattribute__(self, name):
+            if name in missing_fields:
+                raise MissingOriginationError(fake_operation_data.originated_contract_address, missing_fields.keys())
+            return super().__getattr__()
+
+        fake_operation_data.__getattribute__ = MethodType(__getattribute__, fake_operation_data)  # type: ignore
+        return fake_operation_data
+
     @classmethod
     def convert_big_map(cls, big_map_json: Dict[str, Any]) -> BigMapData:
         """Convert raw big map diff message from WS/REST into dataclass"""
diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py
index a5a9a61c2..5ed68214b 100644
--- a/src/dipdup/exceptions.py
+++ b/src/dipdup/exceptions.py
@@ -1,7 +1,7 @@
 import traceback
 from abc import ABC, abstractmethod
 from pprint import pformat
-from typing import Any, Optional, Type
+from typing import Any, Iterable, Optional, Type
 
 from tabulate import tabulate
 
@@ -62,6 +62,11 @@
 {error_context}
 """
 
+_missing_origination_error = """Contract `{address}` has been originated with blockchain migration instead of operation.
+
+The following `OperationData` fields are not available (since there's no real operation): {missing_fields}.
+"""
+
 
 class DipDupError(ABC, Exception):
     exit_code = 1
@@ -175,3 +180,18 @@ def format_help(self) -> str:
             type_name=self.type_name,
             error_context=pformat(self.error_context, compact=True),
         )
+
+
+class MissingOriginationError(DipDupError):
+    """Contract has been originated with blockchain migration"""
+
+    def __init__(self, address: str, missing_fields: Iterable[str]) -> None:
+        super().__init__(None)
+        self.address = address
+        self.missing_fields = missing_fields
+
+    def format_help(self) -> str:
+        return _missing_origination_error.format(
+            address=self.address,
+            missing_fields=', '.join(self.missing_fields),
+        )
diff --git a/src/dipdup/index.py b/src/dipdup/index.py
index 376a0eede..b219e14f9 100644
--- a/src/dipdup/index.py
+++ b/src/dipdup/index.py
@@ -114,9 +114,10 @@ class OperationIndex(Index):
     def __init__(self, ctx: DipDupContext, config: OperationIndexConfig, datasource: TzktDatasource) -> None:
         super().__init__(ctx, config, datasource)
         self._queue: Deque[Tuple[int, List[OperationData], Optional[HeadBlockData]]] = deque()
-        self._contract_hashes: Dict[str, Tuple[str, str]] = {}
+        self._contract_hashes: Dict[str, Tuple[int, int]] = {}
         self._rollback_level: Optional[int] = None
         self._last_hashes: Set[str] = set()
+        self._migration_originations: Optional[Dict[str, OperationData]] = None
 
     def push(self, level: int, operations: List[OperationData], block: Optional[HeadBlockData] = None) -> None:
         self._queue.append((level, operations, block))
@@ -148,6 +149,11 @@ async def _synchronize(self, last_level: int, cache: bool = False) -> None:
         transaction_addresses = await self._get_transaction_addresses()
         origination_addresses = await self._get_origination_addresses()
 
+        migration_originations = await self._datasource.get_migration_originations()
+        for op in migration_originations:
+            code_hash, type_hash = await self._get_contract_hashes(cast(str, op.originated_contract_address))
+            op.originated_contract_code_hash, op.originated_contract_type_hash = code_hash, type_hash
+
         fetcher = OperationFetcher(
             datasource=self._datasource,
             first_level=first_level,
@@ -155,6 +161,7 @@ async def _synchronize(self, last_level: int, cache: bool = False) -> None:
             transaction_addresses=transaction_addresses,
             origination_addresses=origination_addresses,
             cache=cache,
+            migration_originations=migration_originations,
         )
 
         async for level, operations in fetcher.fetch_operations_by_level():
@@ -377,7 +384,7 @@ async def _get_origination_addresses(self) -> Set[str]:
                             addresses.add(address)
         return addresses
 
-    async def _get_contract_hashes(self, address: str) -> Tuple[str, str]:
+    async def _get_contract_hashes(self, address: str) -> Tuple[int, int]:
         if address not in self._contract_hashes:
             summary = await self._datasource.get_contract_summary(address)
             self._contract_hashes[address] = (summary['codeHash'], summary['typeHash'])
diff --git a/src/dipdup/models.py b/src/dipdup/models.py
index cb1b36dc1..3f0647f5c 100644
--- a/src/dipdup/models.py
+++ b/src/dipdup/models.py
@@ -77,6 +77,7 @@ class OperationData:
     entrypoint: Optional[str] = None
     parameter_json: Optional[Any] = None
     originated_contract_address: Optional[str] = None
+    originated_contract_alias: Optional[str] = None
     originated_contract_type_hash: Optional[int] = None
     originated_contract_code_hash: Optional[int] = None
     diffs: Optional[List[Dict[str, Any]]] = None

From c97d854e5710dd2b9398dac1709227a474974e3a Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 14:10:01 +0300
Subject: [PATCH 02/14] Better error handling

---
 src/dipdup/codegen.py                     |  1 +
 src/dipdup/config.py                      | 45 +++++++++++++----------
 src/dipdup/datasources/tzkt/datasource.py | 20 +++-------
 3 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py
index e6910ddac..6a7c0381d 100644
--- a/src/dipdup/codegen.py
+++ b/src/dipdup/codegen.py
@@ -242,6 +242,7 @@ async def generate_types(self) -> None:
                     with open(output_path) as type_file:
                         first_line = type_file.readline()
                         if re.match(r'^#\s+dipdup:\s+ignore\s*', first_line):
+                            self._logger.info('Skipping `%s`', output_path)
                             continue
 
                 if name == 'storage':
diff --git a/src/dipdup/config.py b/src/dipdup/config.py
index ecdf4e4b7..037ea1ee9 100644
--- a/src/dipdup/config.py
+++ b/src/dipdup/config.py
@@ -277,12 +277,15 @@ def storage_type_cls(self, typ: Type) -> None:
 
     def initialize_storage_cls(self, package: str, module_name: str) -> None:
         _logger.info('Registering `%s` storage type', module_name)
-        storage_type_module = importlib.import_module(f'{package}.types.{module_name}.storage')
-        storage_type_cls = getattr(
-            storage_type_module,
-            snake_to_pascal(module_name) + 'Storage',
-        )
-        self.storage_type_cls = storage_type_cls
+        # TODO: import_from util
+        cls_name = snake_to_pascal(module_name) + 'Storage'
+        module_name = f'{package}.types.{module_name}.storage'
+        try:
+            storage_type_module = importlib.import_module(module_name)
+            storage_type_cls = getattr(storage_type_module, cls_name)
+            self.storage_type_cls = storage_type_cls
+        except (ModuleNotFoundError, AttributeError) as e:
+            raise HandlerImportError(module_name, cls_name) from e
 
 
 @dataclass
@@ -322,10 +325,15 @@ def parameter_type_cls(self, typ: Type) -> None:
 
     def initialize_parameter_cls(self, package: str, module_name: str, entrypoint: str) -> None:
         _logger.info('Registering parameter type for entrypoint `%s`', entrypoint)
-        parameter_type_module = importlib.import_module(f'{package}.types.{module_name}.parameter.{pascal_to_snake(entrypoint)}')
-        parameter_type_cls = getattr(parameter_type_module, snake_to_pascal(entrypoint) + 'Parameter')
-        self.parameter_type_cls = parameter_type_cls
-
+        # TODO: import_from util
+        module_name = f'{package}.types.{module_name}.parameter.{pascal_to_snake(entrypoint)}'
+        cls_name = snake_to_pascal(entrypoint) + 'Parameter'
+        try:
+            parameter_type_module = importlib.import_module(module_name)
+            parameter_type_cls = getattr(parameter_type_module, cls_name)
+            self.parameter_type_cls = parameter_type_cls
+        except (ModuleNotFoundError, AttributeError) as e:
+            raise HandlerImportError(module_name, cls_name) from e
 
 @dataclass
 class TransactionIdMixin:
@@ -751,27 +759,21 @@ def validate(self) -> None:
             raise ConfigurationError('SQLite DB engine is not supported by Hasura')
 
     def get_contract(self, name: str) -> ContractConfig:
-        if name.startswith('<') and name.endswith('>'):
-            raise ConfigurationError(f'`{name}` variable of index template is not set')
-
+        self._check_name(name)
         try:
             return self.contracts[name]
         except KeyError as e:
             raise ConfigurationError(f'Contract `{name}` not found in `contracts` config section') from e
 
     def get_datasource(self, name: str) -> DatasourceConfigT:
-        if name.startswith('<') and name.endswith('>'):
-            raise ConfigurationError(f'`{name}` variable of index template is not set')
-
+        self._check_name(name)
         try:
             return self.datasources[name]
         except KeyError as e:
             raise ConfigurationError(f'Datasource `{name}` not found in `datasources` config section') from e
 
     def get_template(self, name: str) -> IndexConfigTemplateT:
-        if name.startswith('<') and name.endswith('>'):
-            raise ConfigurationError(f'`{name}` variable of index template is not set')
-
+        self._check_name(name)
         try:
             return self.templates[name]
         except KeyError as e:
@@ -812,6 +814,11 @@ def resolve_index_templates(self) -> None:
                 new_index_config.parent = index_config.parent
                 self.indexes[index_name] = new_index_config
 
+    def _check_name(self, name: str) -> None:
+        variable = name.split('<')[-1].split('>')[0]
+        if variable != name:
+            raise ConfigurationError(f'`{variable}` variable of index template is not set')
+
     def _pre_initialize_index(self, index_name: str, index_config: IndexConfigT) -> None:
         """Resolve contract and datasource configs by aliases"""
         if index_name in self._pre_initialized:
diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py
index 9d97bc2ef..984b89a13 100644
--- a/src/dipdup/datasources/tzkt/datasource.py
+++ b/src/dipdup/datasources/tzkt/datasource.py
@@ -772,13 +772,6 @@ def convert_migration_origination(cls, migration_origination_json: Dict[str, Any
         if not isinstance(storage, Dict):
             storage = {}
 
-        missing_fields = dict(
-            hash='',
-            counter=0,
-            sender_address='',
-            target_address=None,
-            initiator_address=None,
-        )
         fake_operation_data = OperationData(
             type='origination',
             id=migration_origination_json['id'],
@@ -792,15 +785,12 @@ def convert_migration_origination(cls, migration_origination_json: Dict[str, Any
             diffs=migration_origination_json.get('diffs'),
             status='applied',
             has_internals=False,
-            **missing_fields,  # type: ignore
+            hash='[none]',
+            counter=0,
+            sender_address='[none]',
+            target_address=None,
+            initiator_address=None,
         )
-
-        def __getattribute__(self, name):
-            if name in missing_fields:
-                raise MissingOriginationError(fake_operation_data.originated_contract_address, missing_fields.keys())
-            return super().__getattr__()
-
-        fake_operation_data.__getattribute__ = MethodType(__getattribute__, fake_operation_data)  # type: ignore
         return fake_operation_data
 
     @classmethod

From 87d4bbe87bfbf060550e51369926838535ddf263 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 14:15:52 +0300
Subject: [PATCH 03/14] Bump hasura

---
 src/demo_hic_et_nunc/docker/docker-compose.yml           | 2 +-
 src/demo_quipuswap/docker/docker-compose.yml             | 2 +-
 src/demo_registrydao/docker/docker-compose.yml           | 2 +-
 src/demo_tezos_domains/docker/docker-compose.yml         | 2 +-
 src/demo_tezos_domains_big_map/docker/docker-compose.yml | 2 +-
 src/demo_tzbtc/docker/docker-compose.yml                 | 2 +-
 src/demo_tzcolors/docker/docker-compose.yml              | 2 +-
 src/dipdup/templates/docker/docker-compose.yml.j2        | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/demo_hic_et_nunc/docker/docker-compose.yml b/src/demo_hic_et_nunc/docker/docker-compose.yml
index aa6ddbfc0..41351f722 100644
--- a/src/demo_hic_et_nunc/docker/docker-compose.yml
+++ b/src/demo_hic_et_nunc/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/demo_quipuswap/docker/docker-compose.yml b/src/demo_quipuswap/docker/docker-compose.yml
index cce0cc917..18128f50b 100644
--- a/src/demo_quipuswap/docker/docker-compose.yml
+++ b/src/demo_quipuswap/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/demo_registrydao/docker/docker-compose.yml b/src/demo_registrydao/docker/docker-compose.yml
index d16ef5f41..d5d660349 100644
--- a/src/demo_registrydao/docker/docker-compose.yml
+++ b/src/demo_registrydao/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/demo_tezos_domains/docker/docker-compose.yml b/src/demo_tezos_domains/docker/docker-compose.yml
index d4998374f..b7075d836 100644
--- a/src/demo_tezos_domains/docker/docker-compose.yml
+++ b/src/demo_tezos_domains/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/demo_tezos_domains_big_map/docker/docker-compose.yml b/src/demo_tezos_domains_big_map/docker/docker-compose.yml
index 00393ada0..391d260be 100644
--- a/src/demo_tezos_domains_big_map/docker/docker-compose.yml
+++ b/src/demo_tezos_domains_big_map/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/demo_tzbtc/docker/docker-compose.yml b/src/demo_tzbtc/docker/docker-compose.yml
index e715ebf40..53dc5eefb 100644
--- a/src/demo_tzbtc/docker/docker-compose.yml
+++ b/src/demo_tzbtc/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/demo_tzcolors/docker/docker-compose.yml b/src/demo_tzcolors/docker/docker-compose.yml
index 014d77ac9..57702398a 100644
--- a/src/demo_tzcolors/docker/docker-compose.yml
+++ b/src/demo_tzcolors/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
diff --git a/src/dipdup/templates/docker/docker-compose.yml.j2 b/src/dipdup/templates/docker/docker-compose.yml.j2
index b404144ec..10eb7cd9c 100644
--- a/src/dipdup/templates/docker/docker-compose.yml.j2
+++ b/src/dipdup/templates/docker/docker-compose.yml.j2
@@ -31,7 +31,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

From 4513f502cde50f215769ee12362a7736f6aa38fc Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 14:24:24 +0300
Subject: [PATCH 04/14] Lint, bump deps

---
 poetry.lock                               | 158 ++++++++++------------
 src/dipdup/config.py                      |   1 +
 src/dipdup/datasources/tzkt/datasource.py |   2 -
 src/dipdup/exceptions.py                  |  17 +--
 4 files changed, 76 insertions(+), 102 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index fef13a57f..8e060d7e9 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -54,7 +54,7 @@ typing_extensions = ">=3.7.2"
 
 [[package]]
 name = "anyio"
-version = "3.2.1"
+version = "3.3.0"
 description = "High level compatibility layer for multiple asynchronous event loop implementations"
 category = "main"
 optional = false
@@ -241,11 +241,11 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
 
 [[package]]
 name = "bleach"
-version = "3.3.1"
+version = "4.0.0"
 description = "An easy safelist-based HTML-sanitizing tool."
 category = "main"
 optional = true
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+python-versions = ">=3.6"
 
 [package.dependencies]
 packaging = "*"
@@ -380,7 +380,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [[package]]
 name = "charset-normalizer"
-version = "2.0.3"
+version = "2.0.4"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
 category = "main"
 optional = false
@@ -435,7 +435,7 @@ cython = ["cython"]
 
 [[package]]
 name = "datamodel-code-generator"
-version = "0.11.8"
+version = "0.11.9"
 description = "Datamodel Code Generator"
 category = "main"
 optional = false
@@ -447,7 +447,7 @@ black = ">=19.10b0"
 genson = ">=1.2.1,<2.0"
 inflect = ">=4.1.0,<6.0"
 isort = ">=4.3.21,<6.0"
-jinja2 = ">=2.10.1,<3.0"
+jinja2 = ">=2.10.1,<4.0"
 openapi-spec-validator = ">=0.2.8,<0.4"
 prance = ">=0.18.2,<1.0"
 pydantic = [
@@ -458,13 +458,11 @@ PySnooper = ">=0.4.1,<1.0.0"
 toml = ">=0.10.0,<1.0.0"
 
 [package.extras]
-all = ["pytest-runner", "setuptools-scm", "pytest (>=4.6)", "pytest-benchmark", "pytest-cov", "pytest-mock", "mypy", "isort", "freezegun", "types-jinja2", "types-pyyaml", "types-toml", "httpx", "mkdocs", "mkdocs-material", "wheel", "twine", "codecov"]
+all = ["pytest (>=4.6)", "pytest-benchmark", "pytest-cov", "pytest-mock", "mypy", "isort", "freezegun", "types-jinja2", "types-pyyaml", "types-toml", "httpx", "mkdocs", "mkdocs-material", "codecov"]
 ci = ["codecov"]
 docs = ["mkdocs", "mkdocs-material"]
 http = ["httpx"]
-setup = ["pytest-runner", "setuptools-scm"]
 test = ["pytest (>=4.6)", "pytest-benchmark", "pytest-cov", "pytest-mock", "mypy", "isort", "freezegun", "types-jinja2", "types-pyyaml", "types-toml"]
-wheel = ["wheel", "twine"]
 
 [[package]]
 name = "dateutils"
@@ -729,7 +727,7 @@ test = ["pytest (!=5.3.4)", "pytest-cov", "flaky", "nose", "jedi (<=0.17.2)"]
 
 [[package]]
 name = "ipython"
-version = "7.25.0"
+version = "7.26.0"
 description = "IPython: Productive Interactive Computing"
 category = "main"
 optional = true
@@ -788,7 +786,7 @@ six = "*"
 
 [[package]]
 name = "isort"
-version = "5.9.2"
+version = "5.9.3"
 description = "A Python utility / library to sort Python imports."
 category = "main"
 optional = false
@@ -817,17 +815,17 @@ testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"]
 
 [[package]]
 name = "jinja2"
-version = "2.11.3"
+version = "3.0.1"
 description = "A very fast and expressive template engine."
 category = "main"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+python-versions = ">=3.6"
 
 [package.dependencies]
-MarkupSafe = ">=0.23"
+MarkupSafe = ">=2.0"
 
 [package.extras]
-i18n = ["Babel (>=0.8)"]
+i18n = ["Babel (>=2.7)"]
 
 [[package]]
 name = "jinja2-pluralize"
@@ -974,11 +972,11 @@ python-versions = "*"
 
 [[package]]
 name = "mnemonic"
-version = "0.19"
+version = "0.20"
 description = "Implementation of Bitcoin BIP-0039"
 category = "main"
 optional = true
-python-versions = "*"
+python-versions = ">=3.5"
 
 [[package]]
 name = "monotonic"
@@ -1511,7 +1509,7 @@ testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist",
 
 [[package]]
 name = "pytezos"
-version = "3.2.5"
+version = "3.2.6"
 description = "Python toolkit for Tezos"
 category = "main"
 optional = true
@@ -1636,7 +1634,7 @@ py = {version = "*", markers = "implementation_name == \"pypy\""}
 
 [[package]]
 name = "regex"
-version = "2021.7.6"
+version = "2021.8.3"
 description = "Alternative regular expression module, to replace re."
 category = "main"
 optional = false
@@ -1723,7 +1721,7 @@ win32 = ["pywin32"]
 
 [[package]]
 name = "sentry-sdk"
-version = "1.3.0"
+version = "1.3.1"
 description = "Python client for Sentry (https://sentry.io)"
 category = "main"
 optional = false
@@ -1906,7 +1904,7 @@ accel = ["ciso8601 (>=2.1.2,<3.0.0)", "python-rapidjson", "uvloop (>=0.14.0,<0.1
 
 [[package]]
 name = "tqdm"
-version = "4.61.2"
+version = "4.62.0"
 description = "Fast, Extensible Progress Meter"
 category = "main"
 optional = true
@@ -2118,8 +2116,8 @@ aiosqlite = [
     {file = "aiosqlite-0.16.1.tar.gz", hash = "sha256:2e915463164efa65b60fd1901aceca829b6090082f03082618afca6fb9c8fdf7"},
 ]
 anyio = [
-    {file = "anyio-3.2.1-py3-none-any.whl", hash = "sha256:442678a3c7e1cdcdbc37dcfe4527aa851b1b0c9162653b516e9f509821691d50"},
-    {file = "anyio-3.2.1.tar.gz", hash = "sha256:07968db9fa7c1ca5435a133dc62f988d84ef78e1d9b22814a59d1c62618afbc5"},
+    {file = "anyio-3.3.0-py3-none-any.whl", hash = "sha256:929a6852074397afe1d989002aa96d457e3e1e5441357c60d03e7eea0e65e1b0"},
+    {file = "anyio-3.3.0.tar.gz", hash = "sha256:ae57a67583e5ff8b4af47666ff5651c3732d45fd26c929253748e796af860374"},
 ]
 appdirs = [
     {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
@@ -2210,8 +2208,8 @@ black = [
     {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
 ]
 bleach = [
-    {file = "bleach-3.3.1-py2.py3-none-any.whl", hash = "sha256:ae976d7174bba988c0b632def82fdc94235756edfb14e6558a9c5be555c9fb78"},
-    {file = "bleach-3.3.1.tar.gz", hash = "sha256:306483a5a9795474160ad57fce3ddd1b50551e981eed8e15a582d34cef28aafa"},
+    {file = "bleach-4.0.0-py2.py3-none-any.whl", hash = "sha256:c1685a132e6a9a38bf93752e5faab33a9517a6c0bb2f37b785e47bf253bdb51d"},
+    {file = "bleach-4.0.0.tar.gz", hash = "sha256:ffa9221c6ac29399cc50fcc33473366edd0cf8d5e2cbbbb63296dc327fb67cc8"},
 ]
 bravado = [
     {file = "bravado-11.0.3-py2.py3-none-any.whl", hash = "sha256:8ac8bbb645e49607917a5c07808116c708521f51e80d9c29bc4a168ff4dd22c6"},
@@ -2291,8 +2289,8 @@ chardet = [
     {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"},
 ]
 charset-normalizer = [
-    {file = "charset-normalizer-2.0.3.tar.gz", hash = "sha256:c46c3ace2d744cfbdebceaa3c19ae691f53ae621b39fd7570f59d14fb7f2fd12"},
-    {file = "charset_normalizer-2.0.3-py3-none-any.whl", hash = "sha256:88fce3fa5b1a84fdcb3f603d889f723d1dd89b26059d0123ca435570e848d5e1"},
+    {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"},
+    {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"},
 ]
 click = [
     {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"},
@@ -2360,8 +2358,8 @@ cytoolz = [
     {file = "cytoolz-0.11.0.tar.gz", hash = "sha256:c64f3590c3eb40e1548f0d3c6b2ccde70493d0b8dc6cc7f9f3fec0bb3dcd4222"},
 ]
 datamodel-code-generator = [
-    {file = "datamodel-code-generator-0.11.8.tar.gz", hash = "sha256:5d5d654f4811d3e029fab118a5188337da72e5aea1dfc9d6a689327cfb988171"},
-    {file = "datamodel_code_generator-0.11.8-py3-none-any.whl", hash = "sha256:cd0a90bc81612e1b04b7b60e9321c73dc4e1122f05eaa657e645a03ed41be6c0"},
+    {file = "datamodel-code-generator-0.11.9.tar.gz", hash = "sha256:ee65eaef77be2f366093a7efb1a89940af7dae892606081d8442ecd1dd9c7a6a"},
+    {file = "datamodel_code_generator-0.11.9-py3-none-any.whl", hash = "sha256:419860f753b394ffc542d5af764f1db0c8ac4231707a14863ed5bd18a7d1c04e"},
 ]
 dateutils = [
     {file = "dateutils-0.6.12-py2.py3-none-any.whl", hash = "sha256:f33b6ab430fa4166e7e9cb8b21ee9f6c9843c48df1a964466f52c79b2a8d53b3"},
@@ -2444,8 +2442,8 @@ ipykernel = [
     {file = "ipykernel-5.5.5.tar.gz", hash = "sha256:e976751336b51082a89fc2099fb7f96ef20f535837c398df6eab1283c2070884"},
 ]
 ipython = [
-    {file = "ipython-7.25.0-py3-none-any.whl", hash = "sha256:aa21412f2b04ad1a652e30564fff6b4de04726ce875eab222c8430edc6db383a"},
-    {file = "ipython-7.25.0.tar.gz", hash = "sha256:54bbd1fe3882457aaf28ae060a5ccdef97f212a741754e420028d4ec5c2291dc"},
+    {file = "ipython-7.26.0-py3-none-any.whl", hash = "sha256:892743b65c21ed72b806a3a602cca408520b3200b89d1924f4b3d2cdb3692362"},
+    {file = "ipython-7.26.0.tar.gz", hash = "sha256:0cff04bb042800129348701f7bd68a430a844e8fb193979c08f6c99f28bb735e"},
 ]
 ipython-genutils = [
     {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"},
@@ -2460,16 +2458,16 @@ isodate = [
     {file = "isodate-0.6.0.tar.gz", hash = "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8"},
 ]
 isort = [
-    {file = "isort-5.9.2-py3-none-any.whl", hash = "sha256:eed17b53c3e7912425579853d078a0832820f023191561fcee9d7cae424e0813"},
-    {file = "isort-5.9.2.tar.gz", hash = "sha256:f65ce5bd4cbc6abdfbe29afc2f0245538ab358c14590912df638033f157d555e"},
+    {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"},
+    {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"},
 ]
 jedi = [
     {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"},
     {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"},
 ]
 jinja2 = [
-    {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"},
-    {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"},
+    {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"},
+    {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"},
 ]
 jinja2-pluralize = [
     {file = "jinja2_pluralize-0.3.0-py2.py3-none-any.whl", hash = "sha256:4fec874a591014774d4c66cb7f65314390731bfc57db4c27119db61aa93b2bc4"},
@@ -2552,8 +2550,8 @@ mistune = [
     {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"},
 ]
 mnemonic = [
-    {file = "mnemonic-0.19-py2.py3-none-any.whl", hash = "sha256:a8d78c5100acfa7df9bab6b9db7390831b0e54490934b718ff9efd68f0d731a6"},
-    {file = "mnemonic-0.19.tar.gz", hash = "sha256:4e37eb02b2cbd56a0079cabe58a6da93e60e3e4d6e757a586d9f23d96abea931"},
+    {file = "mnemonic-0.20-py3-none-any.whl", hash = "sha256:acd2168872d0379e7a10873bb3e12bf6c91b35de758135c4fbd1015ef18fafc5"},
+    {file = "mnemonic-0.20.tar.gz", hash = "sha256:7c6fb5639d779388027a77944680aee4870f0fcd09b1e42a5525ee2ce4c625f6"},
 ]
 monotonic = [
     {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"},
@@ -2899,8 +2897,8 @@ pytest-cov = [
     {file = "pytest_cov-2.9.0-py2.py3-none-any.whl", hash = "sha256:c87dfd8465d865655a8213859f1b4749b43448b5fae465cb981e16d52a811424"},
 ]
 pytezos = [
-    {file = "pytezos-3.2.5-py3-none-any.whl", hash = "sha256:bbfcfbb3f562fc0a529fc66051adac873c47f7f70af727d15d9a1f578961e38b"},
-    {file = "pytezos-3.2.5.tar.gz", hash = "sha256:37c48c99ccdedcd212772209d999c2fa4e9647a9b05f5f91dd8966729c58b0f1"},
+    {file = "pytezos-3.2.6-py3-none-any.whl", hash = "sha256:62f8b9f950329f5b6e9185243e9af545bcf583c8821c65346507ce198a20ec8f"},
+    {file = "pytezos-3.2.6.tar.gz", hash = "sha256:0fcc90804440d7882170844c7f498fddaaa2e20abee3abc7c3432688385e2364"},
 ]
 python-dateutil = [
     {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
@@ -3009,47 +3007,39 @@ pyzmq = [
     {file = "pyzmq-22.1.0.tar.gz", hash = "sha256:7040d6dd85ea65703904d023d7f57fab793d7ffee9ba9e14f3b897f34ff2415d"},
 ]
 regex = [
-    {file = "regex-2021.7.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e6a1e5ca97d411a461041d057348e578dc344ecd2add3555aedba3b408c9f874"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:6afe6a627888c9a6cfbb603d1d017ce204cebd589d66e0703309b8048c3b0854"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ccb3d2190476d00414aab36cca453e4596e8f70a206e2aa8db3d495a109153d2"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:ed693137a9187052fc46eedfafdcb74e09917166362af4cc4fddc3b31560e93d"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99d8ab206a5270c1002bfcf25c51bf329ca951e5a169f3b43214fdda1f0b5f0d"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:b85ac458354165405c8a84725de7bbd07b00d9f72c31a60ffbf96bb38d3e25fa"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:3f5716923d3d0bfb27048242a6e0f14eecdb2e2a7fac47eda1d055288595f222"},
-    {file = "regex-2021.7.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5983c19d0beb6af88cb4d47afb92d96751fb3fa1784d8785b1cdf14c6519407"},
-    {file = "regex-2021.7.6-cp36-cp36m-win32.whl", hash = "sha256:c92831dac113a6e0ab28bc98f33781383fe294df1a2c3dfd1e850114da35fd5b"},
-    {file = "regex-2021.7.6-cp36-cp36m-win_amd64.whl", hash = "sha256:791aa1b300e5b6e5d597c37c346fb4d66422178566bbb426dd87eaae475053fb"},
-    {file = "regex-2021.7.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:59506c6e8bd9306cd8a41511e32d16d5d1194110b8cfe5a11d102d8b63cf945d"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:564a4c8a29435d1f2256ba247a0315325ea63335508ad8ed938a4f14c4116a5d"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:59c00bb8dd8775473cbfb967925ad2c3ecc8886b3b2d0c90a8e2707e06c743f0"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:9a854b916806c7e3b40e6616ac9e85d3cdb7649d9e6590653deb5b341a736cec"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:db2b7df831c3187a37f3bb80ec095f249fa276dbe09abd3d35297fc250385694"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:173bc44ff95bc1e96398c38f3629d86fa72e539c79900283afa895694229fe6a"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:15dddb19823f5147e7517bb12635b3c82e6f2a3a6b696cc3e321522e8b9308ad"},
-    {file = "regex-2021.7.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ddeabc7652024803666ea09f32dd1ed40a0579b6fbb2a213eba590683025895"},
-    {file = "regex-2021.7.6-cp37-cp37m-win32.whl", hash = "sha256:f080248b3e029d052bf74a897b9d74cfb7643537fbde97fe8225a6467fb559b5"},
-    {file = "regex-2021.7.6-cp37-cp37m-win_amd64.whl", hash = "sha256:d8bbce0c96462dbceaa7ac4a7dfbbee92745b801b24bce10a98d2f2b1ea9432f"},
-    {file = "regex-2021.7.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edd1a68f79b89b0c57339bce297ad5d5ffcc6ae7e1afdb10f1947706ed066c9c"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:422dec1e7cbb2efbbe50e3f1de36b82906def93ed48da12d1714cabcd993d7f0"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cbe23b323988a04c3e5b0c387fe3f8f363bf06c0680daf775875d979e376bd26"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:0eb2c6e0fcec5e0f1d3bcc1133556563222a2ffd2211945d7b1480c1b1a42a6f"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:1c78780bf46d620ff4fff40728f98b8afd8b8e35c3efd638c7df67be2d5cddbf"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:bc84fb254a875a9f66616ed4538542fb7965db6356f3df571d783f7c8d256edd"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:598c0a79b4b851b922f504f9f39a863d83ebdfff787261a5ed061c21e67dd761"},
-    {file = "regex-2021.7.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:875c355360d0f8d3d827e462b29ea7682bf52327d500a4f837e934e9e4656068"},
-    {file = "regex-2021.7.6-cp38-cp38-win32.whl", hash = "sha256:e586f448df2bbc37dfadccdb7ccd125c62b4348cb90c10840d695592aa1b29e0"},
-    {file = "regex-2021.7.6-cp38-cp38-win_amd64.whl", hash = "sha256:2fe5e71e11a54e3355fa272137d521a40aace5d937d08b494bed4529964c19c4"},
-    {file = "regex-2021.7.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6110bab7eab6566492618540c70edd4d2a18f40ca1d51d704f1d81c52d245026"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4f64fc59fd5b10557f6cd0937e1597af022ad9b27d454e182485f1db3008f417"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:89e5528803566af4df368df2d6f503c84fbfb8249e6631c7b025fe23e6bd0cde"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2366fe0479ca0e9afa534174faa2beae87847d208d457d200183f28c74eaea59"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f9392a4555f3e4cb45310a65b403d86b589adc773898c25a39184b1ba4db8985"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:2bceeb491b38225b1fee4517107b8491ba54fba77cf22a12e996d96a3c55613d"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:f98dc35ab9a749276f1a4a38ab3e0e2ba1662ce710f6530f5b0a6656f1c32b58"},
-    {file = "regex-2021.7.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319eb2a8d0888fa6f1d9177705f341bc9455a2c8aca130016e52c7fe8d6c37a3"},
-    {file = "regex-2021.7.6-cp39-cp39-win32.whl", hash = "sha256:eaf58b9e30e0e546cdc3ac06cf9165a1ca5b3de8221e9df679416ca667972035"},
-    {file = "regex-2021.7.6-cp39-cp39-win_amd64.whl", hash = "sha256:4c9c3155fe74269f61e27617529b7f09552fbb12e44b1189cebbdb24294e6e1c"},
-    {file = "regex-2021.7.6.tar.gz", hash = "sha256:8394e266005f2d8c6f0bc6780001f7afa3ef81a7a2111fa35058ded6fce79e4d"},
+    {file = "regex-2021.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8764a78c5464ac6bde91a8c87dd718c27c1cabb7ed2b4beaf36d3e8e390567f9"},
+    {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4551728b767f35f86b8e5ec19a363df87450c7376d7419c3cac5b9ceb4bce576"},
+    {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:577737ec3d4c195c4aef01b757905779a9e9aee608fa1cf0aec16b5576c893d3"},
+    {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c856ec9b42e5af4fe2d8e75970fcc3a2c15925cbcc6e7a9bcb44583b10b95e80"},
+    {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3835de96524a7b6869a6c710b26c90e94558c31006e96ca3cf6af6751b27dca1"},
+    {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cea56288eeda8b7511d507bbe7790d89ae7049daa5f51ae31a35ae3c05408531"},
+    {file = "regex-2021.8.3-cp36-cp36m-win32.whl", hash = "sha256:a4eddbe2a715b2dd3849afbdeacf1cc283160b24e09baf64fa5675f51940419d"},
+    {file = "regex-2021.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:57fece29f7cc55d882fe282d9de52f2f522bb85290555b49394102f3621751ee"},
+    {file = "regex-2021.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a5c6dbe09aff091adfa8c7cfc1a0e83fdb8021ddb2c183512775a14f1435fe16"},
+    {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff4a8ad9638b7ca52313d8732f37ecd5fd3c8e3aff10a8ccb93176fd5b3812f6"},
+    {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b63e3571b24a7959017573b6455e05b675050bbbea69408f35f3cb984ec54363"},
+    {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fbc20975eee093efa2071de80df7f972b7b35e560b213aafabcec7c0bd00bd8c"},
+    {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14caacd1853e40103f59571f169704367e79fb78fac3d6d09ac84d9197cadd16"},
+    {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bb350eb1060591d8e89d6bac4713d41006cd4d479f5e11db334a48ff8999512f"},
+    {file = "regex-2021.8.3-cp37-cp37m-win32.whl", hash = "sha256:18fdc51458abc0a974822333bd3a932d4e06ba2a3243e9a1da305668bd62ec6d"},
+    {file = "regex-2021.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:026beb631097a4a3def7299aa5825e05e057de3c6d72b139c37813bfa351274b"},
+    {file = "regex-2021.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:16d9eaa8c7e91537516c20da37db975f09ac2e7772a0694b245076c6d68f85da"},
+    {file = "regex-2021.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3905c86cc4ab6d71635d6419a6f8d972cab7c634539bba6053c47354fd04452c"},
+    {file = "regex-2021.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937b20955806381e08e54bd9d71f83276d1f883264808521b70b33d98e4dec5d"},
+    {file = "regex-2021.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:28e8af338240b6f39713a34e337c3813047896ace09d51593d6907c66c0708ba"},
+    {file = "regex-2021.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c09d88a07483231119f5017904db8f60ad67906efac3f1baa31b9b7f7cca281"},
+    {file = "regex-2021.8.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:85f568892422a0e96235eb8ea6c5a41c8ccbf55576a2260c0160800dbd7c4f20"},
+    {file = "regex-2021.8.3-cp38-cp38-win32.whl", hash = "sha256:bf6d987edd4a44dd2fa2723fca2790f9442ae4de2c8438e53fcb1befdf5d823a"},
+    {file = "regex-2021.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:8fe58d9f6e3d1abf690174fd75800fda9bdc23d2a287e77758dc0e8567e38ce6"},
+    {file = "regex-2021.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7976d410e42be9ae7458c1816a416218364e06e162b82e42f7060737e711d9ce"},
+    {file = "regex-2021.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9569da9e78f0947b249370cb8fadf1015a193c359e7e442ac9ecc585d937f08d"},
+    {file = "regex-2021.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459bbe342c5b2dec5c5223e7c363f291558bc27982ef39ffd6569e8c082bdc83"},
+    {file = "regex-2021.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f421e3cdd3a273bace013751c345f4ebeef08f05e8c10757533ada360b51a39"},
+    {file = "regex-2021.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea212df6e5d3f60341aef46401d32fcfded85593af1d82b8b4a7a68cd67fdd6b"},
+    {file = "regex-2021.8.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a3b73390511edd2db2d34ff09aa0b2c08be974c71b4c0505b4a048d5dc128c2b"},
+    {file = "regex-2021.8.3-cp39-cp39-win32.whl", hash = "sha256:f35567470ee6dbfb946f069ed5f5615b40edcbb5f1e6e1d3d2b114468d505fc6"},
+    {file = "regex-2021.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:bfa6a679410b394600eafd16336b2ce8de43e9b13f7fb9247d84ef5ad2b45e91"},
+    {file = "regex-2021.8.3.tar.gz", hash = "sha256:8935937dad2c9b369c3d932b0edbc52a62647c2afb2fafc0c280f14a8bf56a6a"},
 ]
 requests = [
     {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"},
@@ -3106,8 +3096,8 @@ send2trash = [
     {file = "Send2Trash-1.7.1.tar.gz", hash = "sha256:17730aa0a33ab82ed6ca76be3bb25f0433d0014f1ccf63c979bab13a5b9db2b2"},
 ]
 sentry-sdk = [
-    {file = "sentry-sdk-1.3.0.tar.gz", hash = "sha256:5210a712dd57d88d225c1fc3fe3a3626fee493637bcd54e204826cf04b8d769c"},
-    {file = "sentry_sdk-1.3.0-py2.py3-none-any.whl", hash = "sha256:6864dcb6f7dec692635e5518c2a5c80010adf673c70340817f1a1b713d65bb41"},
+    {file = "sentry-sdk-1.3.1.tar.gz", hash = "sha256:ebe99144fa9618d4b0e7617e7929b75acd905d258c3c779edcd34c0adfffe26c"},
+    {file = "sentry_sdk-1.3.1-py2.py3-none-any.whl", hash = "sha256:f33d34c886d0ba24c75ea8885a8b3a172358853c7cbde05979fc99c29ef7bc52"},
 ]
 simplejson = [
     {file = "simplejson-3.17.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:18302970ce341c3626433d4ffbdac19c7cca3d6e2d54b12778bcb8095f695473"},
@@ -3235,8 +3225,8 @@ tortoise-orm = [
     {file = "tortoise_orm-0.17.5-py3-none-any.whl", hash = "sha256:978ec824837b44373fb1b3669d443d823c71b080e39db37db72355fde6cadc24"},
 ]
 tqdm = [
-    {file = "tqdm-4.61.2-py2.py3-none-any.whl", hash = "sha256:5aa445ea0ad8b16d82b15ab342de6b195a722d75fc1ef9934a46bba6feafbc64"},
-    {file = "tqdm-4.61.2.tar.gz", hash = "sha256:8bb94db0d4468fea27d004a0f1d1c02da3cdedc00fe491c0de986b76a04d6b0a"},
+    {file = "tqdm-4.62.0-py2.py3-none-any.whl", hash = "sha256:706dea48ee05ba16e936ee91cb3791cd2ea6da348a0e50b46863ff4363ff4340"},
+    {file = "tqdm-4.62.0.tar.gz", hash = "sha256:3642d483b558eec80d3c831e23953582c34d7e4540db86d9e5ed9dad238dabc6"},
 ]
 traitlets = [
     {file = "traitlets-5.0.5-py3-none-any.whl", hash = "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"},
diff --git a/src/dipdup/config.py b/src/dipdup/config.py
index 037ea1ee9..058356c2a 100644
--- a/src/dipdup/config.py
+++ b/src/dipdup/config.py
@@ -335,6 +335,7 @@ def initialize_parameter_cls(self, package: str, module_name: str, entrypoint: s
         except (ModuleNotFoundError, AttributeError) as e:
             raise HandlerImportError(module_name, cls_name) from e
 
+
 @dataclass
 class TransactionIdMixin:
     """`transaction_id` field"""
diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py
index 984b89a13..dfa590b54 100644
--- a/src/dipdup/datasources/tzkt/datasource.py
+++ b/src/dipdup/datasources/tzkt/datasource.py
@@ -4,7 +4,6 @@
 from datetime import datetime, timezone
 from decimal import Decimal
 from enum import Enum
-from types import MethodType
 from typing import Any, AsyncGenerator, DefaultDict, Dict, List, NoReturn, Optional, Set, Tuple, cast
 
 from aiohttp import ClientResponseError
@@ -23,7 +22,6 @@
 )
 from dipdup.datasources.datasource import IndexDatasource, SubscriptionManager
 from dipdup.datasources.tzkt.enums import TzktMessageType
-from dipdup.exceptions import MissingOriginationError
 from dipdup.models import BigMapAction, BigMapData, BlockData, HeadBlockData, OperationData
 from dipdup.utils import groupby, split_by_chunks
 
diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py
index 5ed68214b..b10fed6fb 100644
--- a/src/dipdup/exceptions.py
+++ b/src/dipdup/exceptions.py
@@ -1,7 +1,7 @@
 import traceback
 from abc import ABC, abstractmethod
 from pprint import pformat
-from typing import Any, Iterable, Optional, Type
+from typing import Any, Optional, Type
 
 from tabulate import tabulate
 
@@ -180,18 +180,3 @@ def format_help(self) -> str:
             type_name=self.type_name,
             error_context=pformat(self.error_context, compact=True),
         )
-
-
-class MissingOriginationError(DipDupError):
-    """Contract has been originated with blockchain migration"""
-
-    def __init__(self, address: str, missing_fields: Iterable[str]) -> None:
-        super().__init__(None)
-        self.address = address
-        self.missing_fields = missing_fields
-
-    def format_help(self) -> str:
-        return _missing_origination_error.format(
-            address=self.address,
-            missing_fields=', '.join(self.missing_fields),
-        )

From dbb9216c622e7088bb7e485b424cb371ac0fad2a Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 15:13:20 +0300
Subject: [PATCH 05/14] import_from

---
 src/dipdup/config.py | 57 +++++++++++++-------------------------------
 src/dipdup/utils.py  | 10 ++++++++
 2 files changed, 26 insertions(+), 41 deletions(-)

diff --git a/src/dipdup/config.py b/src/dipdup/config.py
index 058356c2a..1aef9734b 100644
--- a/src/dipdup/config.py
+++ b/src/dipdup/config.py
@@ -19,8 +19,8 @@
 from ruamel.yaml import YAML
 from typing_extensions import Literal
 
-from dipdup.exceptions import ConfigurationError, HandlerImportError
-from dipdup.utils import pascal_to_snake, snake_to_pascal
+from dipdup.exceptions import ConfigurationError
+from dipdup.utils import import_from, pascal_to_snake, snake_to_pascal
 
 ROLLBACK_HANDLER = 'on_rollback'
 CONFIGURE_HANDLER = 'on_configure'
@@ -128,7 +128,6 @@ def name(self, name: str) -> None:
 class ContractConfig(NameMixin):
     """Contract config
 
-    :param network: Corresponding network alias, only for sanity checks
     :param address: Contract address
     :param typename: User-defined alias for the contract script
     """
@@ -277,15 +276,9 @@ def storage_type_cls(self, typ: Type) -> None:
 
     def initialize_storage_cls(self, package: str, module_name: str) -> None:
         _logger.info('Registering `%s` storage type', module_name)
-        # TODO: import_from util
         cls_name = snake_to_pascal(module_name) + 'Storage'
         module_name = f'{package}.types.{module_name}.storage'
-        try:
-            storage_type_module = importlib.import_module(module_name)
-            storage_type_cls = getattr(storage_type_module, cls_name)
-            self.storage_type_cls = storage_type_cls
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module_name, cls_name) from e
+        self.storage_type_cls = import_from(module_name, cls_name)
 
 
 @dataclass
@@ -325,15 +318,9 @@ def parameter_type_cls(self, typ: Type) -> None:
 
     def initialize_parameter_cls(self, package: str, module_name: str, entrypoint: str) -> None:
         _logger.info('Registering parameter type for entrypoint `%s`', entrypoint)
-        # TODO: import_from util
         module_name = f'{package}.types.{module_name}.parameter.{pascal_to_snake(entrypoint)}'
         cls_name = snake_to_pascal(entrypoint) + 'Parameter'
-        try:
-            parameter_type_module = importlib.import_module(module_name)
-            parameter_type_cls = getattr(parameter_type_module, cls_name)
-            self.parameter_type_cls = parameter_type_cls
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module_name, cls_name) from e
+        self.parameter_type_cls = import_from(module_name, cls_name)
 
 
 @dataclass
@@ -787,18 +774,14 @@ def get_tzkt_datasource(self, name: str) -> TzktDatasourceConfig:
         return datasource
 
     def get_rollback_fn(self) -> Type:
-        try:
-            module_name = f'{self.package}.handlers.{ROLLBACK_HANDLER}'
-            return getattr(importlib.import_module(module_name), ROLLBACK_HANDLER)
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=ROLLBACK_HANDLER) from e
+        module_name = f'{self.package}.handlers.{ROLLBACK_HANDLER}'
+        fn_name = ROLLBACK_HANDLER
+        return import_from(module_name, fn_name)
 
     def get_configure_fn(self) -> Type:
-        try:
-            module_name = f'{self.package}.handlers.{CONFIGURE_HANDLER}'
-            return getattr(importlib.import_module(module_name), CONFIGURE_HANDLER)
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=CONFIGURE_HANDLER) from e
+        module_name = f'{self.package}.handlers.{CONFIGURE_HANDLER}'
+        fn_name = CONFIGURE_HANDLER
+        return import_from(module_name, fn_name)
 
     def resolve_index_templates(self) -> None:
         _logger.info('Substituting index templates')
@@ -956,23 +939,15 @@ def load(
 
     def _initialize_handler_callback(self, handler_config: HandlerConfig) -> None:
         _logger.info('Registering handler callback `%s`', handler_config.callback)
-        try:
-            module_name = f'{self.package}.handlers.{handler_config.callback}'
-            module = importlib.import_module(module_name)
-            callback_fn = getattr(module, handler_config.callback)
-            handler_config.callback_fn = callback_fn
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=handler_config.callback) from e
+        module_name = f'{self.package}.handlers.{handler_config.callback}'
+        fn_name = handler_config.callback
+        handler_config.callback_fn = import_from(module_name, fn_name)
 
     def _initialize_job_callback(self, job_config: JobConfig) -> None:
         _logger.info('Registering job callback `%s`', job_config.callback)
-        try:
-            module_name = f'{self.package}.jobs.{job_config.callback}'
-            module = importlib.import_module(module_name)
-            callback_fn = getattr(module, job_config.callback)
-            job_config.callback_fn = callback_fn
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=job_config.callback) from e
+        module_name = f'{self.package}.jobs.{job_config.callback}'
+        fn_name = job_config.callback
+        job_config.callback_fn = import_from(module_name, fn_name)
 
     def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None:
         if index_name in self._initialized:
diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py
index c5289f3a6..dbea335ea 100644
--- a/src/dipdup/utils.py
+++ b/src/dipdup/utils.py
@@ -23,6 +23,8 @@
 from tortoise.models import Model
 from tortoise.transactions import in_transaction
 
+from dipdup.exceptions import HandlerImportError
+
 _logger = logging.getLogger('dipdup.utils')
 
 
@@ -221,3 +223,11 @@ def write(path: str, content: str, overwrite: bool = False) -> bool:
     with open(path, 'w') as file:
         file.write(content)
     return True
+
+
+def import_from(module: str, obj: str) -> Any:
+    """Import object from module, raise HandlerImportError on failure"""
+    try:
+        return getattr(importlib.import_module(module), obj)
+    except (ImportError, AttributeError) as e:
+        raise HandlerImportError(module, obj) from e

From da6c79bff152507de1c18d209d5c6ac4291ac1e4 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 15:16:33 +0300
Subject: [PATCH 06/14] Cleanup

---
 src/dipdup/exceptions.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py
index b10fed6fb..a5a9a61c2 100644
--- a/src/dipdup/exceptions.py
+++ b/src/dipdup/exceptions.py
@@ -62,11 +62,6 @@
 {error_context}
 """
 
-_missing_origination_error = """Contract `{address}` has been originated with blockchain migration instead of operation.
-
-The following `OperationData` fields are not available (since there's no real operation): {missing_fields}.
-"""
-
 
 class DipDupError(ABC, Exception):
     exit_code = 1

From 407ec5c2cee8dfc8cc2dfe48766410e5e4ab805d Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 15:21:20 +0300
Subject: [PATCH 07/14] plain table in alreadyexisis help messages

---
 src/dipdup/exceptions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py
index a5a9a61c2..541851712 100644
--- a/src/dipdup/exceptions.py
+++ b/src/dipdup/exceptions.py
@@ -144,7 +144,7 @@ def __init__(self, ctx, name: str, address: str) -> None:
         self.address = address
 
     def format_help(self) -> str:
-        contracts_table = tabulate([(name, c.address) for name, c in self.ctx.config.contracts.items()])
+        contracts_table = tabulate([(c.name, c.address) for c in self.ctx.config.contracts.values()], tablefmt='plain')
         return _contract_already_exists_error.format(name=self.name, address=self.address, contracts_table=contracts_table)
 
 
@@ -156,7 +156,7 @@ def __init__(self, ctx, name: str) -> None:
         self.name = name
 
     def format_help(self) -> str:
-        indexes_table = tabulate([(name,) for name in self.ctx.config.indexes.values()])
+        indexes_table = tabulate([(c.name, c.kind) for c in self.ctx.config.indexes.values()], tablefmt='plain')
         return _index_already_exists_error.format(name=self.name, indexes_table=indexes_table)
 
 

From 4b4b430cec04804a60fd36fb617d760a8faec6ec Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Wed, 4 Aug 2021 17:42:53 +0300
Subject: [PATCH 08/14] Ignore past migrations

---
 src/dipdup/datasources/datasource.py      | 11 ++++++++++-
 src/dipdup/datasources/tzkt/datasource.py |  6 +++++-
 src/dipdup/index.py                       |  2 +-
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/dipdup/datasources/datasource.py b/src/dipdup/datasources/datasource.py
index 2856b2f88..77d5073fa 100644
--- a/src/dipdup/datasources/datasource.py
+++ b/src/dipdup/datasources/datasource.py
@@ -3,6 +3,7 @@
 from copy import copy
 from enum import Enum
 from functools import partial
+import logging
 from typing import Awaitable, DefaultDict, List, Optional, Protocol, Set
 
 from pydantic.dataclasses import dataclass
@@ -14,6 +15,10 @@
 from dipdup.models import BigMapData, HeadBlockData, OperationData
 
 
+# NOTE: Since there's no other index datasource
+_logger = logging.getLogger('dipdup.tzkt')
+
+
 class EventType(Enum):
     operations = 'operatitions'
     big_maps = 'big_maps'
@@ -100,12 +105,16 @@ def get_pending(self, active_subscriptions: 'Subscriptions') -> 'Subscriptions':
             big_maps=defaultdict(set, {k: self.big_maps[k] for k in set(self.big_maps) - set(active_subscriptions.big_maps)}),
         )
 
-
 class SubscriptionManager:
     def __init__(self) -> None:
         self._subscriptions: Subscriptions = Subscriptions()
         self._active_subscriptions: Subscriptions = Subscriptions()
 
+    def status(self, pending: bool = False) -> str:
+        subs = self.get_pending() if pending else self._active_subscriptions
+        big_maps_len = sum([len(v) for v in subs.big_maps.values()])
+        return f'{len(subs.address_transactions)} contracts, {int(subs.originations)} originations, {int(subs.head)} head, {big_maps_len} big maps'
+
     def add_address_transaction_subscription(self, address: str) -> None:
         self._subscriptions.address_transactions.add(address)
 
diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py
index dfa590b54..57a5b50b0 100644
--- a/src/dipdup/datasources/tzkt/datasource.py
+++ b/src/dipdup/datasources/tzkt/datasource.py
@@ -395,7 +395,7 @@ async def get_block(self, level: int) -> BlockData:
         )
         return self.convert_block(block_json)
 
-    async def get_migration_originations(self) -> List[OperationData]:
+    async def get_migration_originations(self, first_level: int = 0) -> List[OperationData]:
         """Get contracts originated from migrations"""
         self._logger.info('Fetching contracts originated with migrations')
         # NOTE: Empty unwrapped request to ensure API supports migration originations
@@ -416,6 +416,7 @@ async def get_migration_originations(self) -> List[OperationData]:
             url='v1/operations/migrations',
             params={
                 'kind': 'origination',
+                'level.gt': first_level,
                 'select': ','.join(ORIGINATION_MIGRATION_FIELDS),
             },
         )
@@ -563,6 +564,9 @@ async def subscribe(self) -> None:
             return
 
         pending_subscriptions = self._subscriptions.get_pending()
+        self._logger.info('Subscribing to channels')
+        self._logger.info('Active: %s', self._subscriptions.status(False))
+        self._logger.info('Pending: %s', self._subscriptions.status(True))
 
         for address in pending_subscriptions.address_transactions:
             await self._subscribe_to_address_transactions(address)
diff --git a/src/dipdup/index.py b/src/dipdup/index.py
index b219e14f9..9e89145c3 100644
--- a/src/dipdup/index.py
+++ b/src/dipdup/index.py
@@ -149,7 +149,7 @@ async def _synchronize(self, last_level: int, cache: bool = False) -> None:
         transaction_addresses = await self._get_transaction_addresses()
         origination_addresses = await self._get_origination_addresses()
 
-        migration_originations = await self._datasource.get_migration_originations()
+        migration_originations = await self._datasource.get_migration_originations(first_level)
         for op in migration_originations:
             code_hash, type_hash = await self._get_contract_hashes(cast(str, op.originated_contract_address))
             op.originated_contract_code_hash, op.originated_contract_type_hash = code_hash, type_hash

From 2468746586b43bb8b2e805de41378c90a9ab7aa7 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Thu, 5 Aug 2021 10:48:27 +0300
Subject: [PATCH 09/14] Fix subscriptions

---
 src/demo_tezos_domains/dipdup.yml         | 14 +++++++-------
 src/dipdup/datasources/tzkt/datasource.py |  5 ++++-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/demo_tezos_domains/dipdup.yml b/src/demo_tezos_domains/dipdup.yml
index 59eba74c9..c9f482cff 100644
--- a/src/demo_tezos_domains/dipdup.yml
+++ b/src/demo_tezos_domains/dipdup.yml
@@ -6,14 +6,14 @@ database:
   path: tezos_domains.sqlite3
 
 contracts:
-  edo_name_registry:
-    address: KT1JJbWfW8CHUY95hG9iq2CEMma1RiKhMHDR
+  mainnet_name_registry:
+    address: KT1GBZmSxmnKJXGMdMLbugPfLyUPmuLSMwKS
     typename: name_registry
 
 datasources:
-  tzkt_edo:
+  tzkt_mainnet:
     kind: tzkt
-    url: ${TZKT_URL:-https://api.edo2net.tzkt.io}
+    url: ${TZKT_URL:-https://api.tzkt.io}
 
 templates:
   tezos_domains:
@@ -34,8 +34,8 @@ templates:
             entrypoint: execute
 
 indexes:
-  tezos_domains_edo:
+  tezos_domains_mainnet:
     template: tezos_domains
     values:
-      datasource: tzkt_edo
-      name_registry: edo_name_registry
+      datasource: tzkt_mainnet
+      name_registry: mainnet_name_registry
diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py
index 57a5b50b0..4698ef898 100644
--- a/src/dipdup/datasources/tzkt/datasource.py
+++ b/src/dipdup/datasources/tzkt/datasource.py
@@ -555,9 +555,12 @@ async def run(self) -> None:
     async def _on_connect(self) -> None:
         """Subscribe to all required channels on established WS connection"""
         self._logger.info('Connected to server')
-        self._subscriptions.reset()
         await self.subscribe()
 
+    async def _on_disconnect(self) -> None:
+        self._logger.info('Disconnected from server')
+        self._subscriptions.reset()
+
     async def subscribe(self) -> None:
         """Subscribe to all required channels"""
         if not self._realtime:

From 62049c152c42f057314f2c82e883cf3bd0df2710 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Thu, 5 Aug 2021 12:31:56 +0300
Subject: [PATCH 10/14] Revert `SubscriptionManager`

---
 src/dipdup/datasources/datasource.py      | 60 +----------------------
 src/dipdup/datasources/tzkt/datasource.py | 53 ++++++++------------
 src/dipdup/dipdup.py                      |  3 +-
 tests/integration_tests/test_rollback.py  |  4 +-
 4 files changed, 24 insertions(+), 96 deletions(-)

diff --git a/src/dipdup/datasources/datasource.py b/src/dipdup/datasources/datasource.py
index 77d5073fa..ae588f574 100644
--- a/src/dipdup/datasources/datasource.py
+++ b/src/dipdup/datasources/datasource.py
@@ -1,13 +1,7 @@
 from abc import abstractmethod
-from collections import defaultdict
-from copy import copy
 from enum import Enum
-from functools import partial
-import logging
-from typing import Awaitable, DefaultDict, List, Optional, Protocol, Set
+from typing import Awaitable, List, Optional, Protocol
 
-from pydantic.dataclasses import dataclass
-from pydantic.fields import Field
 from pyee import AsyncIOEventEmitter  # type: ignore
 
 from dipdup.config import HTTPConfig
@@ -15,10 +9,6 @@
 from dipdup.models import BigMapData, HeadBlockData, OperationData
 
 
-# NOTE: Since there's no other index datasource
-_logger = logging.getLogger('dipdup.tzkt')
-
-
 class EventType(Enum):
     operations = 'operatitions'
     big_maps = 'big_maps'
@@ -88,51 +78,3 @@ def emit_rollback(self, from_level: int, to_level: int) -> None:
 
     def emit_head(self, block: HeadBlockData) -> None:
         super().emit(EventType.head, datasource=self, block=block)
-
-
-@dataclass
-class Subscriptions:
-    address_transactions: Set[str] = Field(default_factory=set)
-    originations: bool = False
-    head: bool = False
-    big_maps: DefaultDict[str, Set[str]] = Field(default_factory=partial(defaultdict, set))
-
-    def get_pending(self, active_subscriptions: 'Subscriptions') -> 'Subscriptions':
-        return Subscriptions(
-            address_transactions=self.address_transactions.difference(active_subscriptions.address_transactions),
-            originations=not active_subscriptions.originations,
-            head=not active_subscriptions.head,
-            big_maps=defaultdict(set, {k: self.big_maps[k] for k in set(self.big_maps) - set(active_subscriptions.big_maps)}),
-        )
-
-class SubscriptionManager:
-    def __init__(self) -> None:
-        self._subscriptions: Subscriptions = Subscriptions()
-        self._active_subscriptions: Subscriptions = Subscriptions()
-
-    def status(self, pending: bool = False) -> str:
-        subs = self.get_pending() if pending else self._active_subscriptions
-        big_maps_len = sum([len(v) for v in subs.big_maps.values()])
-        return f'{len(subs.address_transactions)} contracts, {int(subs.originations)} originations, {int(subs.head)} head, {big_maps_len} big maps'
-
-    def add_address_transaction_subscription(self, address: str) -> None:
-        self._subscriptions.address_transactions.add(address)
-
-    def add_origination_subscription(self) -> None:
-        self._subscriptions.originations = True
-
-    def add_head_subscription(self) -> None:
-        self._subscriptions.head = True
-
-    def add_big_map_subscription(self, address: str, paths: Set[str]) -> None:
-        self._subscriptions.big_maps[address] = self._subscriptions.big_maps[address] | paths
-
-    def get_pending(self) -> Subscriptions:
-        pending_subscriptions = self._subscriptions.get_pending(self._active_subscriptions)
-        return pending_subscriptions
-
-    def commit(self) -> None:
-        self._active_subscriptions = copy(self._subscriptions)
-
-    def reset(self) -> None:
-        self._active_subscriptions = Subscriptions()
diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py
index 4698ef898..751422769 100644
--- a/src/dipdup/datasources/tzkt/datasource.py
+++ b/src/dipdup/datasources/tzkt/datasource.py
@@ -20,7 +20,7 @@
     OperationHandlerOriginationPatternConfig,
     OperationIndexConfig,
 )
-from dipdup.datasources.datasource import IndexDatasource, SubscriptionManager
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.datasources.tzkt.enums import TzktMessageType
 from dipdup.models import BigMapAction, BigMapData, BlockData, HeadBlockData, OperationData
 from dipdup.utils import groupby, split_by_chunks
@@ -292,13 +292,13 @@ def __init__(
         self,
         url: str,
         http_config: Optional[HTTPConfig] = None,
-        realtime: bool = True,
     ) -> None:
         super().__init__(url, http_config)
         self._logger = logging.getLogger('dipdup.tzkt')
-        self._subscriptions: SubscriptionManager = SubscriptionManager()
 
-        self._realtime: bool = realtime
+        self._transaction_subscriptions: Set[str] = set()
+        self._origination_subscriptions: bool = False
+        self._big_map_subscriptions: Dict[str, Set[str]] = {}
         self._client: Optional[BaseHubConnection] = None
 
         self._block: Optional[HeadBlockData] = None
@@ -502,22 +502,24 @@ async def add_index(self, index_config: IndexConfigTemplateT) -> None:
 
         if isinstance(index_config, OperationIndexConfig):
             for contract_config in index_config.contracts or []:
-                self._subscriptions.add_address_transaction_subscription(cast(ContractConfig, contract_config).address)
-
+                self._transaction_subscriptions.add(cast(ContractConfig, contract_config).address)
             for handler_config in index_config.handlers:
                 for pattern_config in handler_config.pattern:
                     if isinstance(pattern_config, OperationHandlerOriginationPatternConfig):
-                        self._subscriptions.add_origination_subscription()
+                        self._origination_subscriptions = True
 
         elif isinstance(index_config, BigMapIndexConfig):
             for big_map_handler_config in index_config.handlers:
                 address, path = big_map_handler_config.contract_config.address, big_map_handler_config.path
-                self._subscriptions.add_big_map_subscription(address, set(path))
+                if address not in self._big_map_subscriptions:
+                    self._big_map_subscriptions[address] = set()
+                if path not in self._big_map_subscriptions[address]:
+                    self._big_map_subscriptions[address].add(path)
 
         else:
             raise NotImplementedError(f'Index kind `{index_config.kind}` is not supported')
 
-        await self.subscribe()
+        await self._on_connect()
 
     def _get_client(self) -> BaseHubConnection:
         """Create SignalR client, register message callbacks"""
@@ -554,34 +556,19 @@ async def run(self) -> None:
 
     async def _on_connect(self) -> None:
         """Subscribe to all required channels on established WS connection"""
-        self._logger.info('Connected to server')
-        await self.subscribe()
-
-    async def _on_disconnect(self) -> None:
-        self._logger.info('Disconnected from server')
-        self._subscriptions.reset()
-
-    async def subscribe(self) -> None:
-        """Subscribe to all required channels"""
-        if not self._realtime:
+        if self._get_client().transport.state != ConnectionState.connected:
             return
 
-        pending_subscriptions = self._subscriptions.get_pending()
-        self._logger.info('Subscribing to channels')
-        self._logger.info('Active: %s', self._subscriptions.status(False))
-        self._logger.info('Pending: %s', self._subscriptions.status(True))
-
-        for address in pending_subscriptions.address_transactions:
-            await self._subscribe_to_address_transactions(address)
-        if pending_subscriptions.originations:
+        self._logger.info('Connected to server')
+        await self._subscribe_to_head()
+        for address in self._transaction_subscriptions:
+            await self._subscribe_to_transactions(address)
+        # NOTE: All originations are passed to matcher
+        if self._origination_subscriptions:
             await self._subscribe_to_originations()
-        if pending_subscriptions.head:
-            await self._subscribe_to_head()
-        for address, paths in pending_subscriptions.big_maps.items():
+        for address, paths in self._big_map_subscriptions.items():
             await self._subscribe_to_big_maps(address, paths)
 
-        self._subscriptions.commit()
-
     # NOTE: Pay attention: this is not a pyee callback
     def _on_error(self, message: CompletionMessage) -> NoReturn:
         """Raise exception from WS server's error message"""
@@ -589,7 +576,7 @@ def _on_error(self, message: CompletionMessage) -> NoReturn:
 
     # TODO: Catch exceptions from pyee 'error' channel
 
-    async def _subscribe_to_address_transactions(self, address: str) -> None:
+    async def _subscribe_to_transactions(self, address: str) -> None:
         """Subscribe to contract's operations on established WS connection"""
         self._logger.info('Subscribing to %s transactions', address)
         await self._send(
diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py
index 73af61933..298000eb3 100644
--- a/src/dipdup/dipdup.py
+++ b/src/dipdup/dipdup.py
@@ -188,7 +188,7 @@ def __init__(self, config: DipDupConfig) -> None:
 
     async def init(self) -> None:
         """Create new or update existing dipdup project"""
-        await self._create_datasources(realtime=False)
+        await self._create_datasources()
 
         async with AsyncExitStack() as stack:
             for datasource in self._datasources.values():
@@ -271,7 +271,6 @@ async def _create_datasources(self, realtime: bool = True) -> None:
                 datasource = TzktDatasource(
                     url=datasource_config.url,
                     http_config=datasource_config.http,
-                    realtime=realtime,
                 )
             elif isinstance(datasource_config, BcdDatasourceConfig):
                 datasource = BcdDatasource(
diff --git a/tests/integration_tests/test_rollback.py b/tests/integration_tests/test_rollback.py
index f34ceb563..aeee9ac25 100644
--- a/tests/integration_tests/test_rollback.py
+++ b/tests/integration_tests/test_rollback.py
@@ -94,7 +94,7 @@ async def test_rollback_ok(self):
         config.database.path = ':memory:'
 
         datasource_name, datasource_config = list(config.datasources.items())[0]
-        datasource = TzktDatasource('test', realtime=False)
+        datasource = TzktDatasource('test')
         dipdup = DipDup(config)
         dipdup._datasources[datasource_name] = datasource
         dipdup._datasources_by_config[datasource_config] = datasource
@@ -121,7 +121,7 @@ async def test_rollback_fail(self):
         config.database.path = ':memory:'
 
         datasource_name, datasource_config = list(config.datasources.items())[0]
-        datasource = TzktDatasource('test', realtime=False)
+        datasource = TzktDatasource('test')
         dipdup = DipDup(config)
         dipdup._datasources[datasource_name] = datasource
         dipdup._datasources_by_config[datasource_config] = datasource

From fd87b27a0b7a283409bc9945eb7aa0ce61583712 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Thu, 5 Aug 2021 13:56:47 +0300
Subject: [PATCH 11/14] Separate type for migrations

---
 src/dipdup/config.py |  1 +
 src/dipdup/index.py  | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/dipdup/config.py b/src/dipdup/config.py
index 1aef9734b..e0f97ed10 100644
--- a/src/dipdup/config.py
+++ b/src/dipdup/config.py
@@ -36,6 +36,7 @@
 class OperationType(Enum):
     transaction = 'transaction'
     origination = 'origination'
+    migration = 'migration'
 
 
 @dataclass
diff --git a/src/dipdup/index.py b/src/dipdup/index.py
index 9e89145c3..7f9b02e30 100644
--- a/src/dipdup/index.py
+++ b/src/dipdup/index.py
@@ -149,10 +149,12 @@ async def _synchronize(self, last_level: int, cache: bool = False) -> None:
         transaction_addresses = await self._get_transaction_addresses()
         origination_addresses = await self._get_origination_addresses()
 
-        migration_originations = await self._datasource.get_migration_originations(first_level)
-        for op in migration_originations:
-            code_hash, type_hash = await self._get_contract_hashes(cast(str, op.originated_contract_address))
-            op.originated_contract_code_hash, op.originated_contract_type_hash = code_hash, type_hash
+        migration_originations = []
+        if self._config.types and OperationType.migration in self._config.types:
+            migration_originations = await self._datasource.get_migration_originations(first_level)
+            for op in migration_originations:
+                code_hash, type_hash = await self._get_contract_hashes(cast(str, op.originated_contract_address))
+                op.originated_contract_code_hash, op.originated_contract_type_hash = code_hash, type_hash
 
         fetcher = OperationFetcher(
             datasource=self._datasource,

From 95042bc80d944caaa729b87c881b8728f9d0465c Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Thu, 5 Aug 2021 14:45:55 +0300
Subject: [PATCH 12/14] Fix FormattedLogger

---
 src/dipdup/utils.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py
index dbea335ea..de779d9e0 100644
--- a/src/dipdup/utils.py
+++ b/src/dipdup/utils.py
@@ -165,20 +165,21 @@ def groupby(seq: Sequence[_T], key: Callable[[Any], _TT]) -> DefaultDict[_TT, Li
 
 
 class FormattedLogger(Logger):
-    def __init__(
-        self,
-        name: str,
-        fmt: Optional[str] = None,
-    ):
-        logger = logging.getLogger(name)
-        self.__class__ = type(FormattedLogger.__name__, (self.__class__, logger.__class__), {})
-        self.__dict__ = logger.__dict__
+    """Logger wrapper with additional formatting"""
+
+    def __init__(self, name: str, fmt: Optional[str] = None) -> None:
+        self.logger = logging.getLogger(name)
         self.fmt = fmt
 
+    def __getattr__(self, name: str) -> Callable:
+        if name == '_log':
+            return self._log
+        return getattr(self.logger, name)
+
     def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1):
         if self.fmt:
             msg = self.fmt.format(msg)
-        super()._log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+        self.logger._log(level, msg, args, exc_info, extra, stack_info, stacklevel)
 
 
 def iter_files(path: str, ext: Optional[str] = None) -> Iterator[TextIO]:

From fbfa5b3c066e2e8e7283d43970369d7abea2dbb3 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Thu, 5 Aug 2021 14:56:40 +0300
Subject: [PATCH 13/14] Prefix datasource logs

---
 src/dipdup/datasources/datasource.py | 3 +++
 src/dipdup/dipdup.py                 | 1 +
 2 files changed, 4 insertions(+)

diff --git a/src/dipdup/datasources/datasource.py b/src/dipdup/datasources/datasource.py
index ae588f574..6c8088dbf 100644
--- a/src/dipdup/datasources/datasource.py
+++ b/src/dipdup/datasources/datasource.py
@@ -1,5 +1,6 @@
 from abc import abstractmethod
 from enum import Enum
+from logging import Logger
 from typing import Awaitable, List, Optional, Protocol
 
 from pyee import AsyncIOEventEmitter  # type: ignore
@@ -37,6 +38,8 @@ def __call__(self, datasource: 'IndexDatasource', block: HeadBlockData) -> Await
 
 
 class Datasource(HTTPGateway):
+    _logger: Logger
+
     @abstractmethod
     async def run(self) -> None:
         ...
diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py
index 298000eb3..ae6bd0aff 100644
--- a/src/dipdup/dipdup.py
+++ b/src/dipdup/dipdup.py
@@ -285,6 +285,7 @@ async def _create_datasources(self, realtime: bool = True) -> None:
             else:
                 raise NotImplementedError
 
+            datasource._logger = FormattedLogger(datasource._logger.name, datasource_config.name + ': {}')
             datasource.set_user_agent(self._config.package)
             self._datasources[name] = datasource
             self._datasources_by_config[datasource_config] = datasource

From 953866693d057be77b3b9705f142c4c6a9e09ff8 Mon Sep 17 00:00:00 2001
From: Lev Gorodetskiy <github@droserasprout.space>
Date: Thu, 5 Aug 2021 15:23:31 +0300
Subject: [PATCH 14/14] Fix block hash mismatch error

---
 src/dipdup/index.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/dipdup/index.py b/src/dipdup/index.py
index 7f9b02e30..ca40ec23e 100644
--- a/src/dipdup/index.py
+++ b/src/dipdup/index.py
@@ -170,6 +170,8 @@ async def _synchronize(self, last_level: int, cache: bool = False) -> None:
             await self._process_level_operations(level, operations)
 
         state.level = last_level  # type: ignore
+        # FIXME: Block hashes are not available during synchronization
+        state.hash = (await self._datasource.get_block(last_level)).hash  # type: ignore
         await state.save()
         self._logger.info('Index is synchronized to level %s', last_level)
 
@@ -439,6 +441,8 @@ async def _synchronize(self, last_level: int, cache: bool = False) -> None:
             await self._process_level_big_maps(level, big_maps)
 
         state.level = last_level  # type: ignore
+        # FIXME: Block hashes are not available during synchronization
+        state.hash = (await self._datasource.get_block(last_level)).hash  # type: ignore
         await state.save()
         self._logger.info('Index is synchronized to level %s', last_level)