Process contracts originated from migrations

src/demo_hic_et_nunc/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/demo_quipuswap/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/demo_registrydao/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/demo_tezos_domains/dipdup.yml

@@ -6,14 +6,14 @@ database:
   path: tezos_domains.sqlite3
 
 contracts:
-  edo_name_registry:
-    address: KT1JJbWfW8CHUY95hG9iq2CEMma1RiKhMHDR
+  mainnet_name_registry:
+    address: KT1GBZmSxmnKJXGMdMLbugPfLyUPmuLSMwKS
     typename: name_registry
 
 datasources:
-  tzkt_edo:
+  tzkt_mainnet:
     kind: tzkt
-    url: ${TZKT_URL:-https://api.edo2net.tzkt.io}
+    url: ${TZKT_URL:-https://api.tzkt.io}
 
 templates:
   tezos_domains:
@@ -34,8 +34,8 @@ templates:
             entrypoint: execute
 
 indexes:
-  tezos_domains_edo:
+  tezos_domains_mainnet:
     template: tezos_domains
     values:
-      datasource: tzkt_edo
-      name_registry: edo_name_registry
+      datasource: tzkt_mainnet
+      name_registry: mainnet_name_registry

src/demo_tezos_domains/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/demo_tezos_domains_big_map/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/demo_tzbtc/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/demo_tzcolors/docker/docker-compose.yml

@@ -30,7 +30,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v2.0.1
+    image: hasura/graphql-engine:v2.0.4
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

src/dipdup/codegen.py

@@ -242,6 +242,7 @@ async def generate_types(self) -> None:
                     with open(output_path) as type_file:
                         first_line = type_file.readline()
                         if re.match(r'^#\s+dipdup:\s+ignore\s*', first_line):
+                            self._logger.info('Skipping `%s`', output_path)
                             continue
 
                 if name == 'storage':

src/dipdup/config.py

@@ -19,8 +19,8 @@
 from ruamel.yaml import YAML
 from typing_extensions import Literal
 
-from dipdup.exceptions import ConfigurationError, HandlerImportError
-from dipdup.utils import pascal_to_snake, snake_to_pascal
+from dipdup.exceptions import ConfigurationError
+from dipdup.utils import import_from, pascal_to_snake, snake_to_pascal
 
 ROLLBACK_HANDLER = 'on_rollback'
 CONFIGURE_HANDLER = 'on_configure'
@@ -36,6 +36,7 @@
 class OperationType(Enum):
     transaction = 'transaction'
     origination = 'origination'
+    migration = 'migration'
 
 
 @dataclass
@@ -128,7 +129,6 @@ def name(self, name: str) -> None:
 class ContractConfig(NameMixin):
     """Contract config
 
-    :param network: Corresponding network alias, only for sanity checks
     :param address: Contract address
     :param typename: User-defined alias for the contract script
     """
@@ -277,12 +277,9 @@ def storage_type_cls(self, typ: Type) -> None:
 
     def initialize_storage_cls(self, package: str, module_name: str) -> None:
         _logger.info('Registering `%s` storage type', module_name)
-        storage_type_module = importlib.import_module(f'{package}.types.{module_name}.storage')
-        storage_type_cls = getattr(
-            storage_type_module,
-            snake_to_pascal(module_name) + 'Storage',
-        )
-        self.storage_type_cls = storage_type_cls
+        cls_name = snake_to_pascal(module_name) + 'Storage'
+        module_name = f'{package}.types.{module_name}.storage'
+        self.storage_type_cls = import_from(module_name, cls_name)
 
 
 @dataclass
@@ -322,9 +319,9 @@ def parameter_type_cls(self, typ: Type) -> None:
 
     def initialize_parameter_cls(self, package: str, module_name: str, entrypoint: str) -> None:
         _logger.info('Registering parameter type for entrypoint `%s`', entrypoint)
-        parameter_type_module = importlib.import_module(f'{package}.types.{module_name}.parameter.{pascal_to_snake(entrypoint)}')
-        parameter_type_cls = getattr(parameter_type_module, snake_to_pascal(entrypoint) + 'Parameter')
-        self.parameter_type_cls = parameter_type_cls
+        module_name = f'{package}.types.{module_name}.parameter.{pascal_to_snake(entrypoint)}'
+        cls_name = snake_to_pascal(entrypoint) + 'Parameter'
+        self.parameter_type_cls = import_from(module_name, cls_name)
 
 
 @dataclass
@@ -751,27 +748,21 @@ def validate(self) -> None:
             raise ConfigurationError('SQLite DB engine is not supported by Hasura')
 
     def get_contract(self, name: str) -> ContractConfig:
-        if name.startswith('<') and name.endswith('>'):
-            raise ConfigurationError(f'`{name}` variable of index template is not set')
-
+        self._check_name(name)
         try:
             return self.contracts[name]
         except KeyError as e:
             raise ConfigurationError(f'Contract `{name}` not found in `contracts` config section') from e
 
     def get_datasource(self, name: str) -> DatasourceConfigT:
-        if name.startswith('<') and name.endswith('>'):
-            raise ConfigurationError(f'`{name}` variable of index template is not set')
-
+        self._check_name(name)
         try:
             return self.datasources[name]
         except KeyError as e:
             raise ConfigurationError(f'Datasource `{name}` not found in `datasources` config section') from e
 
     def get_template(self, name: str) -> IndexConfigTemplateT:
-        if name.startswith('<') and name.endswith('>'):
-            raise ConfigurationError(f'`{name}` variable of index template is not set')
-
+        self._check_name(name)
         try:
             return self.templates[name]
         except KeyError as e:
@@ -784,18 +775,14 @@ def get_tzkt_datasource(self, name: str) -> TzktDatasourceConfig:
         return datasource
 
     def get_rollback_fn(self) -> Type:
-        try:
-            module_name = f'{self.package}.handlers.{ROLLBACK_HANDLER}'
-            return getattr(importlib.import_module(module_name), ROLLBACK_HANDLER)
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=ROLLBACK_HANDLER) from e
+        module_name = f'{self.package}.handlers.{ROLLBACK_HANDLER}'
+        fn_name = ROLLBACK_HANDLER
+        return import_from(module_name, fn_name)
 
     def get_configure_fn(self) -> Type:
-        try:
-            module_name = f'{self.package}.handlers.{CONFIGURE_HANDLER}'
-            return getattr(importlib.import_module(module_name), CONFIGURE_HANDLER)
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=CONFIGURE_HANDLER) from e
+        module_name = f'{self.package}.handlers.{CONFIGURE_HANDLER}'
+        fn_name = CONFIGURE_HANDLER
+        return import_from(module_name, fn_name)
 
     def resolve_index_templates(self) -> None:
         _logger.info('Substituting index templates')
@@ -812,6 +799,11 @@ def resolve_index_templates(self) -> None:
                 new_index_config.parent = index_config.parent
                 self.indexes[index_name] = new_index_config
 
+    def _check_name(self, name: str) -> None:
+        variable = name.split('<')[-1].split('>')[0]
+        if variable != name:
+            raise ConfigurationError(f'`{variable}` variable of index template is not set')
+
     def _pre_initialize_index(self, index_name: str, index_config: IndexConfigT) -> None:
         """Resolve contract and datasource configs by aliases"""
         if index_name in self._pre_initialized:
@@ -948,23 +940,15 @@ def load(
 
     def _initialize_handler_callback(self, handler_config: HandlerConfig) -> None:
         _logger.info('Registering handler callback `%s`', handler_config.callback)
-        try:
-            module_name = f'{self.package}.handlers.{handler_config.callback}'
-            module = importlib.import_module(module_name)
-            callback_fn = getattr(module, handler_config.callback)
-            handler_config.callback_fn = callback_fn
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=handler_config.callback) from e
+        module_name = f'{self.package}.handlers.{handler_config.callback}'
+        fn_name = handler_config.callback
+        handler_config.callback_fn = import_from(module_name, fn_name)
 
     def _initialize_job_callback(self, job_config: JobConfig) -> None:
         _logger.info('Registering job callback `%s`', job_config.callback)
-        try:
-            module_name = f'{self.package}.jobs.{job_config.callback}'
-            module = importlib.import_module(module_name)
-            callback_fn = getattr(module, job_config.callback)
-            job_config.callback_fn = callback_fn
-        except (ModuleNotFoundError, AttributeError) as e:
-            raise HandlerImportError(module=module_name, obj=job_config.callback) from e
+        module_name = f'{self.package}.jobs.{job_config.callback}'
+        fn_name = job_config.callback
+        job_config.callback_fn = import_from(module_name, fn_name)
 
     def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None:
         if index_name in self._initialized:

src/dipdup/datasources/datasource.py

@@ -1,12 +1,8 @@
 from abc import abstractmethod
-from collections import defaultdict
-from copy import copy
 from enum import Enum
-from functools import partial
-from typing import Awaitable, DefaultDict, List, Optional, Protocol, Set
+from logging import Logger
+from typing import Awaitable, List, Optional, Protocol
 
-from pydantic.dataclasses import dataclass
-from pydantic.fields import Field
 from pyee import AsyncIOEventEmitter  # type: ignore
 
 from dipdup.config import HTTPConfig
@@ -42,6 +38,8 @@ def __call__(self, datasource: 'IndexDatasource', block: HeadBlockData) -> Await
 
 
 class Datasource(HTTPGateway):
+    _logger: Logger
+
     @abstractmethod
     async def run(self) -> None:
         ...
@@ -83,47 +81,3 @@ def emit_rollback(self, from_level: int, to_level: int) -> None:
 
     def emit_head(self, block: HeadBlockData) -> None:
         super().emit(EventType.head, datasource=self, block=block)
-
-
-@dataclass
-class Subscriptions:
-    address_transactions: Set[str] = Field(default_factory=set)
-    originations: bool = False
-    head: bool = False
-    big_maps: DefaultDict[str, Set[str]] = Field(default_factory=partial(defaultdict, set))
-
-    def get_pending(self, active_subscriptions: 'Subscriptions') -> 'Subscriptions':
-        return Subscriptions(
-            address_transactions=self.address_transactions.difference(active_subscriptions.address_transactions),
-            originations=not active_subscriptions.originations,
-            head=not active_subscriptions.head,
-            big_maps=defaultdict(set, {k: self.big_maps[k] for k in set(self.big_maps) - set(active_subscriptions.big_maps)}),
-        )
-
-
-class SubscriptionManager:
-    def __init__(self) -> None:
-        self._subscriptions: Subscriptions = Subscriptions()
-        self._active_subscriptions: Subscriptions = Subscriptions()
-
-    def add_address_transaction_subscription(self, address: str) -> None:
-        self._subscriptions.address_transactions.add(address)
-
-    def add_origination_subscription(self) -> None:
-        self._subscriptions.originations = True
-
-    def add_head_subscription(self) -> None:
-        self._subscriptions.head = True
-
-    def add_big_map_subscription(self, address: str, paths: Set[str]) -> None:
-        self._subscriptions.big_maps[address] = self._subscriptions.big_maps[address] | paths
-
-    def get_pending(self) -> Subscriptions:
-        pending_subscriptions = self._subscriptions.get_pending(self._active_subscriptions)
-        return pending_subscriptions
-
-    def commit(self) -> None:
-        self._active_subscriptions = copy(self._subscriptions)
-
-    def reset(self) -> None:
-        self._active_subscriptions = Subscriptions()