Fix processing empty SQL and GraphQL files, refactor codegen

src/dipdup/codegen.py

@@ -3,10 +3,11 @@
 import os
 import re
 import subprocess
+from contextlib import suppress
 from copy import copy
 from os.path import basename, dirname, exists, join, relpath, splitext
 from shutil import rmtree
-from typing import Any, Dict, cast
+from typing import Any, Dict, List, Optional, cast
 
 from jinja2 import Template
 
@@ -18,10 +19,11 @@
     ContractConfig,
     DatasourceConfigT,
     DipDupConfig,
+    IndexConfigT,
+    IndexTemplateConfig,
     OperationHandlerOriginationPatternConfig,
     OperationHandlerTransactionPatternConfig,
     OperationIndexConfig,
-    StaticTemplateConfig,
     TzktDatasourceConfig,
 )
 from dipdup.datasources import DatasourceT
@@ -121,13 +123,30 @@ async def create_package(self) -> None:
         graphql_path = join(self._config.package_path, 'graphql')
         touch(join(graphql_path, '.keep'))
 
-    async def fetch_schemas(self) -> None:
+    async def fetch_schemas(self, template: Optional[str] = None, contract: Optional[str] = None) -> None:
         """Fetch JSONSchemas for all contracts used in config"""
         self._logger.info('Creating `schemas` package')
         schemas_path = join(self._config.package_path, 'schemas')
         mkdir_p(schemas_path)
 
-        for index_config in self._config.indexes.values():
+        index_configs: List[IndexConfigT]
+        if template:
+            if not contract:
+                raise RuntimeError('Both `template` and `contract` arguments are required')
+            # FIXME: Implement classmethods to avoid adding temporary index
+            self._config.indexes['TEMP'] = IndexTemplateConfig(
+                template=template,
+                # NOTE: Regex magic! Replace all variables. What could possibly go wrong?
+                values={'\w*': contract},
+            )
+            self._config.resolve_index_templates()
+            self._config.pre_initialize()
+            index_config = self._config.indexes.pop('TEMP')
+            index_configs = [index_config]
+        else:
+            index_configs = list(self._config.indexes.values())
+
+        for index_config in index_configs:
 
             if isinstance(index_config, OperationIndexConfig):
                 for operation_handler_config in index_config.handlers:
@@ -157,7 +176,20 @@ async def fetch_schemas(self) -> None:
                         storage_schema = resolve_big_maps(contract_schemas['storageSchema'])
                         write(storage_schema_path, json.dumps(storage_schema, indent=4, sort_keys=True))
 
-                        if not isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig):
+                        if isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig):
+                            pass
+                        elif (
+                            isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig)
+                            and operation_pattern_config.similar_to
+                        ):
+                            contract_name = operation_pattern_config.similar_to_contract_config.name
+                            for template in self._config.templates:
+
+                                # NOTE: We don't know which template will be used in factory handler, so let's try all
+                                with suppress(ConfigurationError):
+                                    await self.fetch_schemas(template=template, contract=contract_name)
+                            continue
+                        else:
                             continue
 
                         parameter_schemas_path = join(contract_schemas_path, 'parameter')
@@ -208,7 +240,7 @@ async def fetch_schemas(self) -> None:
                     big_map_value_schema_path = join(big_map_schemas_path, f'{big_map_path}_value.json')
                     write(big_map_value_schema_path, json.dumps(big_map_value_schema, indent=4))
 
-            elif isinstance(index_config, StaticTemplateConfig):
+            elif isinstance(index_config, IndexTemplateConfig):
                 raise RuntimeError('Config is not pre-initialized')
 
             else:

src/dipdup/config.py

@@ -109,7 +109,23 @@ def merge(self, other: Optional['HTTPConfig']) -> None:
 
 
 @dataclass
-class ContractConfig:
+class NameMixin:
+    def __post_init_post_parse__(self) -> None:
+        self._name: Optional[str] = None
+
+    @property
+    def name(self) -> str:
+        if self._name is None:
+            raise RuntimeError('Config is not pre-initialized')
+        return self._name
+
+    @name.setter
+    def name(self, name: str) -> None:
+        self._name = name
+
+
+@dataclass
+class ContractConfig(NameMixin):
     """Contract config
 
     :param network: Corresponding network alias, only for sanity checks
@@ -135,22 +151,6 @@ def valid_address(cls, v):
         return v
 
 
-@dataclass
-class NameMixin:
-    def __post_init_post_parse__(self) -> None:
-        self._name: Optional[str] = None
-
-    @property
-    def name(self) -> str:
-        if self._name is None:
-            raise RuntimeError('Config is not pre-initialized')
-        return self._name
-
-    @name.setter
-    def name(self, name: str) -> None:
-        self._name = name
-
-
 @dataclass
 class TzktDatasourceConfig(NameMixin):
     """TzKT datasource config
@@ -625,13 +625,13 @@ class BlockIndexConfig(IndexConfig):
 
 
 @dataclass
-class StaticTemplateConfig:
+class IndexTemplateConfig:
     kind = 'template'
     template: str
     values: Dict[str, str]
 
 
-IndexConfigT = Union[OperationIndexConfig, BigMapIndexConfig, BlockIndexConfig, StaticTemplateConfig]
+IndexConfigT = Union[OperationIndexConfig, BigMapIndexConfig, BlockIndexConfig, IndexTemplateConfig]
 IndexConfigTemplateT = Union[OperationIndexConfig, BigMapIndexConfig, BlockIndexConfig]
 HandlerPatternConfigT = Union[OperationHandlerOriginationPatternConfig, OperationHandlerTransactionPatternConfig]
 
@@ -707,7 +707,7 @@ class DipDupConfig:
     datasources: Dict[str, DatasourceConfigT]
     contracts: Dict[str, ContractConfig] = Field(default_factory=dict)
     indexes: Dict[str, IndexConfigT] = Field(default_factory=dict)
-    templates: Optional[Dict[str, IndexConfigTemplateT]] = None
+    templates: Dict[str, IndexConfigTemplateT] = Field(default_factory=dict)
     database: Union[SqliteDatabaseConfig, PostgresDatabaseConfig] = SqliteDatabaseConfig(kind='sqlite')
     hasura: Optional[HasuraConfig] = None
     jobs: Optional[Dict[str, JobConfig]] = None
@@ -773,10 +773,10 @@ def get_configure_fn(self) -> Type:
         except (ModuleNotFoundError, AttributeError) as e:
             raise HandlerImportError(module=module_name, obj=CONFIGURE_HANDLER) from e
 
-    def resolve_static_templates(self) -> None:
+    def resolve_index_templates(self) -> None:
         _logger.info('Substituting index templates')
         for index_name, index_config in self.indexes.items():
-            if isinstance(index_config, StaticTemplateConfig):
+            if isinstance(index_config, IndexTemplateConfig):
                 template = self.get_template(index_config.template)
                 raw_template = json.dumps(template, default=pydantic_encoder)
                 for key, value in index_config.values.items():
@@ -840,10 +840,12 @@ def _pre_initialize_index(self, index_name: str, index_config: IndexConfigT) ->
         self._pre_initialized.append(index_name)
 
     def pre_initialize(self) -> None:
-        for name, config in self.datasources.items():
-            config.name = name
+        for name, contract_config in self.contracts.items():
+            contract_config.name = name
+        for name, datasource_config in self.datasources.items():
+            datasource_config.name = name
 
-        self.resolve_static_templates()
+        self.resolve_index_templates()
         for index_name, index_config in self.indexes.items():
             self._pre_initialize_index(index_name, index_config)
 
@@ -941,7 +943,7 @@ def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None
         if index_name in self._initialized:
             return
 
-        if isinstance(index_config, StaticTemplateConfig):
+        if isinstance(index_config, IndexTemplateConfig):
             raise RuntimeError('Config is not pre-initialized')
 
         if isinstance(index_config, OperationIndexConfig):

src/dipdup/context.py

@@ -6,7 +6,7 @@
 from tortoise import Tortoise
 from tortoise.transactions import in_transaction
 
-from dipdup.config import ContractConfig, DipDupConfig, PostgresDatabaseConfig, StaticTemplateConfig
+from dipdup.config import ContractConfig, DipDupConfig, IndexTemplateConfig, PostgresDatabaseConfig
 from dipdup.datasources import DatasourceT
 from dipdup.exceptions import ContractAlreadyExistsError, IndexAlreadyExistsError
 from dipdup.utils import FormattedLogger
@@ -98,7 +98,7 @@ def add_index(self, name: str, template: str, values: Dict[str, Any]) -> None:
         if name in self.config.indexes:
             raise IndexAlreadyExistsError(self, name)
         self.config.get_template(template)
-        self.config.indexes[name] = StaticTemplateConfig(
+        self.config.indexes[name] = IndexTemplateConfig(
             template=template,
             values=values,
         )

src/dipdup/dipdup.py

@@ -21,9 +21,9 @@
     DatasourceConfigT,
     DipDupConfig,
     IndexConfigTemplateT,
+    IndexTemplateConfig,
     OperationIndexConfig,
     PostgresDatabaseConfig,
-    StaticTemplateConfig,
     TzktDatasourceConfig,
 )
 from dipdup.context import DipDupContext, RollbackHandlerContext
@@ -83,7 +83,7 @@ async def reload_config(self) -> None:
         self._ctx.config.initialize()
 
         for index_config in self._ctx.config.indexes.values():
-            if isinstance(index_config, StaticTemplateConfig):
+            if isinstance(index_config, IndexTemplateConfig):
                 raise RuntimeError
             await self.add_index(index_config)