From 68736219cbc265044764658e9c60575374aa7249 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 27 Jul 2021 10:36:41 +0300 Subject: [PATCH 1/8] Skip empty SQL files --- src/dipdup/dipdup.py | 13 ++++--------- src/dipdup/utils.py | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index a952e888d..8713c2d59 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -36,7 +36,7 @@ from dipdup.hasura import HasuraGateway from dipdup.index import BigMapIndex, Index, OperationIndex from dipdup.models import BigMapData, HeadBlockData, IndexType, OperationData, State -from dipdup.utils import FormattedLogger, slowdown, tortoise_wrapper +from dipdup.utils import FormattedLogger, iter_files, slowdown, tortoise_wrapper INDEX_DISPATCHER_INTERVAL = 1.0 from dipdup.scheduler import add_job, create_scheduler @@ -336,14 +336,9 @@ async def _execute_sql_scripts(self, reindex: bool) -> None: if not exists(sql_path): return self._logger.info('Executing SQL scripts from `%s`', sql_path) - for filename in sorted(listdir(sql_path)): - if not filename.endswith('.sql'): - continue - - with open(join(sql_path, filename)) as file: - sql = file.read() - - self._logger.info('Executing `%s`', filename) + for file in iter_files(sql_path, '.sql'): + self._logger.info('Executing `%s`', file.name) + sql = file.read() await get_connection(None).execute_script(sql) def _finish_migration(self, version: str) -> None: diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py index 80c27fb0d..75bd0a49c 100644 --- a/src/dipdup/utils.py +++ b/src/dipdup/utils.py @@ -1,13 +1,17 @@ import asyncio import decimal +import errno +from os import makedirs +from os.path import exists, join, getsize import importlib import logging import pkgutil +from posix import listdir import time import types from contextlib import asynccontextmanager from logging import Logger -from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Type +from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, TextIO, Tuple, Type import humps # type: ignore from tortoise import Tortoise @@ -159,3 +163,25 @@ def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, st if self.fmt: msg = self.fmt.format(msg) super()._log(level, msg, args, exc_info, extra, stack_info, stacklevel) + +def iter_files(path: str, ext: Optional[str] = None) -> Iterator[TextIO]: + """Iterate over files in a directory. Sort alphabetically, filter by extension, skip empty files.""" + if not exists(path): + raise StopIteration + for filename in sorted(listdir(path)): + filepath = join(path, filename) + if ext and not filename.endswith(ext): + continue + if not getsize(filepath): + continue + + with open(filepath) as file: + yield file + +def mkdir_p(path: str) -> None: + """Create directory tree, ignore if already exists""" + try: + makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise From 343cc1beae3c769d884162f26387e30d88cd5000 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 27 Jul 2021 12:06:47 +0300 Subject: [PATCH 2/8] Fix loading empty files, refactor codegen --- src/demo_hic_et_nunc/dipdup-local.yml | 2 +- src/dipdup/codegen.py | 155 ++++++++------------------ src/dipdup/hasura.py | 14 +-- src/dipdup/utils.py | 26 ++++- 4 files changed, 72 insertions(+), 125 deletions(-) diff --git a/src/demo_hic_et_nunc/dipdup-local.yml b/src/demo_hic_et_nunc/dipdup-local.yml index ba3a897b2..3b2972283 100644 --- a/src/demo_hic_et_nunc/dipdup-local.yml +++ b/src/demo_hic_et_nunc/dipdup-local.yml @@ -1,7 +1,7 @@ database: kind: postgres host: 127.0.0.1 - port: 6423 + port: 5432 user: ${POSTGRES_USER:-dipdup} password: ${POSTGRES_PASSWORD:-changeme} database: ${POSTGRES_DB:-dipdup} diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index a90771c1f..fae9ed59f 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -3,9 +3,7 @@ import os import re import subprocess -from contextlib import suppress from copy import copy -from os import mkdir from os.path import basename, dirname, exists, join, relpath, splitext from shutil import rmtree from typing import Any, Dict, cast @@ -29,7 +27,7 @@ from dipdup.datasources import DatasourceT from dipdup.datasources.tzkt.datasource import TzktDatasource from dipdup.exceptions import ConfigurationError -from dipdup.utils import import_submodules, pascal_to_snake, snake_to_pascal +from dipdup.utils import import_submodules, mkdir_p, pascal_to_snake, snake_to_pascal, touch, write DEFAULT_DOCKER_ENV_FILE_CONTENT = dict( POSTGRES_USER="dipdup", @@ -61,6 +59,12 @@ def resolve_big_maps(schema: Dict[str, Any]) -> Dict[str, Any]: return schema +def load_template(name: str) -> Template: + """Load template from templates/{name}.j2""" + with open(join(dirname(__file__), 'templates', name + '.j2'), 'r') as f: + return Template(f.read()) + + class DipDupCodeGenerator: """Generates package based on config, invoked from `init` CLI command""" @@ -91,61 +95,37 @@ async def create_package(self) -> None: package_path = self._config.package_path except ImportError: package_path = join(os.getcwd(), self._config.package) - mkdir(package_path) - with open(join(package_path, '__init__.py'), 'w'): - pass + touch(join(package_path, '__init__.py')) self._logger.info('Creating `%s.models` module', self._config.package) models_path = join(package_path, 'models.py') if not exists(models_path): - with open(join(dirname(__file__), 'templates', 'models.py.j2')) as file: - template = Template(file.read()) + template = load_template('models.py') models_code = template.render() - with open(models_path, 'w') as file: - file.write(models_code) + write(models_path, models_code) self._logger.info('Creating `%s.handlers` package', self._config.package) handlers_path = join(self._config.package_path, 'handlers') - with suppress(FileExistsError): - mkdir(handlers_path) - with open(join(handlers_path, '__init__.py'), 'w'): - pass + touch(join(handlers_path, '__init__.py')) self._logger.info('Creating `%s.jobs` package', self._config.package) jobs_path = join(self._config.package_path, 'jobs') - with suppress(FileExistsError): - mkdir(jobs_path) - with open(join(jobs_path, '__init__.py'), 'w'): - pass + touch(join(jobs_path, '__init__.py')) self._logger.info('Creating `%s/sql` directory', self._config.package) sql_path = join(self._config.package_path, 'sql') - with suppress(FileExistsError): - mkdir(sql_path) - sql_on_restart_path = join(sql_path, 'on_restart') - with suppress(FileExistsError): - mkdir(sql_on_restart_path) - with open(join(sql_on_restart_path, '.keep'), 'w'): - pass - sql_on_reindex_path = join(sql_path, 'on_reindex') - with suppress(FileExistsError): - mkdir(sql_on_reindex_path) - with open(join(sql_on_reindex_path, '.keep'), 'w'): - pass + touch(join(sql_path, 'on_restart', '.keep')) + touch(join(sql_path, 'on_reindex', '.keep')) self._logger.info('Creating `%s/graphql` directory', self._config.package) graphql_path = join(self._config.package_path, 'graphql') - with suppress(FileExistsError): - mkdir(graphql_path) - with open(join(graphql_path, '.keep'), 'w'): - pass + touch(join(graphql_path, '.keep')) async def fetch_schemas(self) -> None: """Fetch JSONSchemas for all contracts used in config""" self._logger.info('Creating `schemas` package') schemas_path = join(self._config.package_path, 'schemas') - with suppress(FileExistsError): - mkdir(schemas_path) + mkdir_p(schemas_path) for index_config in self._config.indexes.values(): @@ -169,8 +149,7 @@ async def fetch_schemas(self) -> None: contract_schemas = await self._get_schema(index_config.datasource_config, contract_config, originated) contract_schemas_path = join(schemas_path, contract_config.module_name) - with suppress(FileExistsError): - mkdir(contract_schemas_path) + mkdir_p(contract_schemas_path) storage_schema_path = join(contract_schemas_path, 'storage.json') @@ -184,8 +163,7 @@ async def fetch_schemas(self) -> None: parameter_schemas_path = join(contract_schemas_path, 'parameter') entrypoint = cast(str, operation_pattern_config.entrypoint) - with suppress(FileExistsError): - mkdir(parameter_schemas_path) + mkdir_p(parameter_schemas_path) try: entrypoint_schema = next( @@ -215,12 +193,10 @@ async def fetch_schemas(self) -> None: contract_schemas = await self._get_schema(index_config.datasource_config, contract_config, False) contract_schemas_path = join(schemas_path, contract_config.module_name) - with suppress(FileExistsError): - mkdir(contract_schemas_path) + mkdir_p(contract_schemas_path) big_map_schemas_path = join(contract_schemas_path, 'big_map') - with suppress(FileExistsError): - mkdir(big_map_schemas_path) + mkdir_p(big_map_schemas_path) try: big_map_schema = next(ep for ep in contract_schemas['bigMaps'] if ep['path'] == big_map_handler_config.path) @@ -255,20 +231,14 @@ async def generate_types(self) -> None: types_path = join(self._config.package_path, 'types') self._logger.info('Creating `types` package') - with suppress(FileExistsError): - mkdir(types_path) - with open(join(types_path, '__init__.py'), 'w'): - pass + touch(join(types_path, '__init__.py')) for root, dirs, files in os.walk(schemas_path): types_root = root.replace(schemas_path, types_path) for dir in dirs: dir_path = join(types_root, dir) - with suppress(FileExistsError): - os.mkdir(dir_path) - with open(join(dir_path, '__init__.py'), 'w'): - pass + touch(join(dir_path, '__init__.py')) for file in files: name, ext = splitext(basename(file)) @@ -278,6 +248,7 @@ async def generate_types(self) -> None: input_path = join(root, file) output_path = join(types_root, f'{pascal_to_snake(name)}.py') + # NOTE: Skip if the first line starts with "# dipdup: ignore" if exists(output_path): with open(output_path) as type_file: first_line = type_file.readline() @@ -307,32 +278,18 @@ async def generate_types(self) -> None: async def generate_default_handlers(self, recreate=False) -> None: handlers_path = join(self._config.package_path, 'handlers') - with open(join(dirname(__file__), 'templates', 'handlers', f'{ROLLBACK_HANDLER}.py.j2')) as file: - rollback_template = Template(file.read()) - with open(join(dirname(__file__), 'templates', 'handlers', f'{CONFIGURE_HANDLER}.py.j2')) as file: - configure_template = Template(file.read()) - - self._logger.info('Generating handler `%s`', CONFIGURE_HANDLER) - handler_code = configure_template.render() - handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py') - if not exists(handler_path) or recreate: - with open(handler_path, 'w') as file: - file.write(handler_code) - - self._logger.info('Generating handler `%s`', ROLLBACK_HANDLER) - handler_code = rollback_template.render() - handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py') - if not exists(handler_path) or recreate: - with open(handler_path, 'w') as file: - file.write(handler_code) + for handler_name in (ROLLBACK_HANDLER, CONFIGURE_HANDLER): + self._logger.info('Generating handler `%s`', handler_name) + template = load_template(f'handlers/{handler_name}.py') + handler_code = template.render() + handler_path = join(handlers_path, f'{handler_name}.py') + write(handler_path, handler_code, overwrite=recreate) async def generate_user_handlers(self) -> None: """Generate handler stubs with typehints from templates if not exist""" handlers_path = join(self._config.package_path, 'handlers') - with open(join(dirname(__file__), 'templates', 'handlers', 'operation.py.j2')) as file: - operation_handler_template = Template(file.read()) - with open(join(dirname(__file__), 'templates', 'handlers', 'big_map.py.j2')) as file: - big_map_handler_template = Template(file.read()) + operation_handler_template = load_template('handlers/operation.py') + big_map_handler_template = load_template('handlers/big_map.py') for index_config in self._config.indexes.values(): if isinstance(index_config, OperationIndexConfig): @@ -346,9 +303,7 @@ async def generate_user_handlers(self) -> None: pascal_to_snake=pascal_to_snake, ) handler_path = join(handlers_path, f'{handler_config.callback}.py') - if not exists(handler_path): - with open(handler_path, 'w') as file: - file.write(handler_code) + write(handler_path, handler_code) elif isinstance(index_config, BigMapIndexConfig): for big_map_handler_config in index_config.handlers: @@ -362,9 +317,7 @@ async def generate_user_handlers(self) -> None: pascal_to_snake=pascal_to_snake, ) handler_path = join(handlers_path, f'{big_map_handler_config.callback}.py') - if not exists(handler_path): - with open(handler_path, 'w') as file: - file.write(handler_code) + write(handler_path, handler_code) else: raise NotImplementedError(f'Index kind `{index_config.kind}` is not supported') @@ -374,39 +327,30 @@ async def generate_jobs(self) -> None: return jobs_path = join(self._config.package_path, 'jobs') - with open(join(dirname(__file__), 'templates', 'jobs', 'job.py.j2')) as file: - job_template = Template(file.read()) + job_template = load_template('jobs/job.py') job_callbacks = set(job_config.callback for job_config in self._config.jobs.values()) for job_callback in job_callbacks: self._logger.info('Generating job `%s`', job_callback) job_code = job_template.render(job=job_callback) job_path = join(jobs_path, f'{job_callback}.py') - if not exists(job_path): - with open(job_path, 'w') as file: - file.write(job_code) + write(job_path, job_code) async def generate_docker(self, image: str, tag: str, env_file: str) -> None: self._logger.info('Generating Docker template') docker_path = join(self._config.package_path, 'docker') - with suppress(FileExistsError): - mkdir(docker_path) + mkdir_p(docker_path) - with open(join(dirname(__file__), 'templates', 'docker', 'Dockerfile.j2')) as file: - dockerfile_template = Template(file.read()) - with open(join(dirname(__file__), 'templates', 'docker', 'docker-compose.yml.j2')) as file: - docker_compose_template = Template(file.read()) - with open(join(dirname(__file__), 'templates', 'docker', 'dipdup.env.j2')) as file: - dipdup_env_template = Template(file.read()) + dockerfile_template = load_template('docker/Dockerfile') + docker_compose_template = load_template('docker/docker-compose.yml') + dipdup_env_template = load_template('docker/dipdup.env') dockerfile_code = dockerfile_template.render( image=f'{image}:{tag}', package=self._config.package, package_path=self._config.package_path, ) - dockerfile_path = join(docker_path, 'Dockerfile') - with open(dockerfile_path, 'w') as file: - file.write(dockerfile_code) + write(join(docker_path, 'Dockerfile'), dockerfile_code, overwrite=True) mounts = {} for filename in self._config.filenames: @@ -426,9 +370,7 @@ async def generate_docker(self, image: str, tag: str, env_file: str) -> None: env_file=env_file, command=command, ) - docker_compose_path = join(docker_path, 'docker-compose.yml') - with open(docker_compose_path, 'w') as file: - file.write(docker_compose_code) + write(join(docker_path, 'docker-compose.yml'), docker_compose_code, overwrite=True) dipdup_env_code = dipdup_env_template.render( environment={ @@ -436,17 +378,10 @@ async def generate_docker(self, image: str, tag: str, env_file: str) -> None: **self._config.environment, } ) - dipdup_env_example_path = join(docker_path, f'{env_file}.example') - with open(dipdup_env_example_path, 'w') as file: - file.write(dipdup_env_code) - dipdup_env_path = join(docker_path, env_file) - if not exists(dipdup_env_path): - with open(dipdup_env_path, 'w') as file: - file.write(dipdup_env_code) - - gitignore_path = join(docker_path, '.gitignore') - with open(gitignore_path, 'w') as file: - file.write('*.env') + write(join(docker_path, 'dipdup.env.example'), dipdup_env_code, overwrite=True) + write(join(docker_path, 'dipdup.env'), dipdup_env_code, overwrite=False) + + write(join(docker_path, '.gitignore'), '*.env') async def cleanup(self) -> None: """Remove fetched JSONSchemas""" diff --git a/src/dipdup/hasura.py b/src/dipdup/hasura.py index 6c9f31f9c..ff7eb15c2 100644 --- a/src/dipdup/hasura.py +++ b/src/dipdup/hasura.py @@ -2,13 +2,11 @@ import importlib import logging from contextlib import suppress -from os import listdir from os.path import dirname, join from typing import Any, Dict, Iterator, List, Optional, Tuple import humps # type: ignore from aiohttp import ClientConnectorError, ClientOSError -from genericpath import exists from pydantic.dataclasses import dataclass from tortoise import fields from tortoise.transactions import get_connection @@ -16,7 +14,7 @@ from dipdup.config import HasuraConfig, HTTPConfig, PostgresDatabaseConfig, pascal_to_snake from dipdup.exceptions import ConfigurationError from dipdup.http import HTTPGateway -from dipdup.utils import iter_models +from dipdup.utils import iter_files, iter_models @dataclass @@ -230,14 +228,8 @@ def _iterate_graphql_queries(self) -> Iterator[Tuple[str, str]]: package = importlib.import_module(self._package) package_path = dirname(package.__file__) graphql_path = join(package_path, 'graphql') - if not exists(graphql_path): - return - for filename in sorted(listdir(graphql_path)): - if not filename.endswith('.graphql'): - continue - - with open(join(graphql_path, filename)) as file: - yield filename[:-8], file.read() + for file in iter_files(graphql_path, '.graphql'): + yield file.name.split('/')[-1][:-8], file.read() async def _generate_query_collections_metadata(self) -> List[Dict[str, Any]]: queries = [] diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py index 75bd0a49c..6539aa0b5 100644 --- a/src/dipdup/utils.py +++ b/src/dipdup/utils.py @@ -1,16 +1,15 @@ import asyncio import decimal import errno -from os import makedirs -from os.path import exists, join, getsize import importlib import logging import pkgutil -from posix import listdir import time import types from contextlib import asynccontextmanager from logging import Logger +from os import listdir, makedirs +from os.path import dirname, exists, getsize, join from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, TextIO, Tuple, Type import humps # type: ignore @@ -164,6 +163,7 @@ def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, st msg = self.fmt.format(msg) super()._log(level, msg, args, exc_info, extra, stack_info, stacklevel) + def iter_files(path: str, ext: Optional[str] = None) -> Iterator[TextIO]: """Iterate over files in a directory. Sort alphabetically, filter by extension, skip empty files.""" if not exists(path): @@ -178,6 +178,7 @@ def iter_files(path: str, ext: Optional[str] = None) -> Iterator[TextIO]: with open(filepath) as file: yield file + def mkdir_p(path: str) -> None: """Create directory tree, ignore if already exists""" try: @@ -185,3 +186,22 @@ def mkdir_p(path: str) -> None: except OSError as e: if e.errno != errno.EEXIST: raise + + +def touch(path: str) -> None: + """Create empty file, ignore if already exists""" + mkdir_p(dirname(path)) + try: + open(path, 'a').close() + except IOError as e: + if e.errno != errno.EEXIST: + raise + + +def write(path: str, content: str, overwrite: bool = False) -> None: + """Write content to file, create directory tree if necessary""" + mkdir_p(dirname(path)) + if exists(path) and not overwrite: + return + with open(path, 'w') as file: + file.write(content) From c0336ab7ebb955c83177c90db1aa3fcbd8d7b512 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 27 Jul 2021 12:17:26 +0300 Subject: [PATCH 3/8] Refactoring --- src/dipdup/codegen.py | 23 ++++++----------------- src/dipdup/utils.py | 5 +++-- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index fae9ed59f..5084b7dcb 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -143,6 +143,7 @@ async def fetch_schemas(self) -> None: contract_config = operation_pattern_config.contract_config originated = bool(operation_pattern_config.source) else: + # NOTE: Operations without entrypoint are untyped continue self._logger.debug(contract_config) @@ -154,9 +155,7 @@ async def fetch_schemas(self) -> None: storage_schema_path = join(contract_schemas_path, 'storage.json') storage_schema = resolve_big_maps(contract_schemas['storageSchema']) - if not exists(storage_schema_path): - with open(storage_schema_path, 'w') as file: - file.write(json.dumps(storage_schema, indent=4, sort_keys=True)) + write(storage_schema_path, json.dumps(storage_schema, indent=4, sort_keys=True)) if not isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): continue @@ -174,11 +173,8 @@ async def fetch_schemas(self) -> None: entrypoint = entrypoint.replace('.', '_').lstrip('_') entrypoint_schema_path = join(parameter_schemas_path, f'{entrypoint}.json') - - if not exists(entrypoint_schema_path): - with open(entrypoint_schema_path, 'w') as file: - file.write(json.dumps(entrypoint_schema, indent=4)) - elif contract_config.typename is not None: + written = write(entrypoint_schema_path, json.dumps(entrypoint_schema, indent=4)) + if not written and contract_config.typename is not None: with open(entrypoint_schema_path, 'r') as file: existing_schema = json.loads(file.read()) if entrypoint_schema != existing_schema: @@ -194,7 +190,6 @@ async def fetch_schemas(self) -> None: contract_schemas_path = join(schemas_path, contract_config.module_name) mkdir_p(contract_schemas_path) - big_map_schemas_path = join(contract_schemas_path, 'big_map') mkdir_p(big_map_schemas_path) @@ -207,17 +202,11 @@ async def fetch_schemas(self) -> None: big_map_path = big_map_handler_config.path.replace('.', '_') big_map_key_schema = big_map_schema['keySchema'] big_map_key_schema_path = join(big_map_schemas_path, f'{big_map_path}_key.json') - - if not exists(big_map_key_schema_path): - with open(big_map_key_schema_path, 'w') as file: - file.write(json.dumps(big_map_key_schema, indent=4)) + write(big_map_key_schema_path, json.dumps(big_map_key_schema, indent=4)) big_map_value_schema = big_map_schema['valueSchema'] big_map_value_schema_path = join(big_map_schemas_path, f'{big_map_path}_value.json') - - if not exists(big_map_value_schema_path): - with open(big_map_value_schema_path, 'w') as file: - file.write(json.dumps(big_map_value_schema, indent=4)) + write(big_map_value_schema_path, json.dumps(big_map_value_schema, indent=4)) elif isinstance(index_config, StaticTemplateConfig): raise RuntimeError('Config is not pre-initialized') diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py index 6539aa0b5..fe5b5c5fe 100644 --- a/src/dipdup/utils.py +++ b/src/dipdup/utils.py @@ -198,10 +198,11 @@ def touch(path: str) -> None: raise -def write(path: str, content: str, overwrite: bool = False) -> None: +def write(path: str, content: str, overwrite: bool = False) -> bool: """Write content to file, create directory tree if necessary""" mkdir_p(dirname(path)) if exists(path) and not overwrite: - return + return False with open(path, 'w') as file: file.write(content) + return True From b5703b5b9b0627f85e028f9bc687a11bf228fc50 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 27 Jul 2021 13:31:02 +0300 Subject: [PATCH 4/8] Fix similar_to init --- src/dipdup/codegen.py | 44 ++++++++++++++++++++++++++++++----- src/dipdup/config.py | 54 ++++++++++++++++++++++--------------------- src/dipdup/context.py | 4 ++-- src/dipdup/dipdup.py | 4 ++-- 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index 5084b7dcb..e6e50ee45 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -3,10 +3,11 @@ import os import re import subprocess +from contextlib import suppress from copy import copy from os.path import basename, dirname, exists, join, relpath, splitext from shutil import rmtree -from typing import Any, Dict, cast +from typing import Any, Dict, List, Optional, cast from jinja2 import Template @@ -18,10 +19,11 @@ ContractConfig, DatasourceConfigT, DipDupConfig, + IndexConfigT, + IndexTemplateConfig, OperationHandlerOriginationPatternConfig, OperationHandlerTransactionPatternConfig, OperationIndexConfig, - StaticTemplateConfig, TzktDatasourceConfig, ) from dipdup.datasources import DatasourceT @@ -121,13 +123,30 @@ async def create_package(self) -> None: graphql_path = join(self._config.package_path, 'graphql') touch(join(graphql_path, '.keep')) - async def fetch_schemas(self) -> None: + async def fetch_schemas(self, template: Optional[str] = None, contract: Optional[str] = None) -> None: """Fetch JSONSchemas for all contracts used in config""" self._logger.info('Creating `schemas` package') schemas_path = join(self._config.package_path, 'schemas') mkdir_p(schemas_path) - for index_config in self._config.indexes.values(): + index_configs: List[IndexConfigT] + if template: + if not contract: + raise RuntimeError('Both `template` and `contract` arguments are required') + # FIXME: Implement classmethods to avoid adding temporary index + self._config.indexes['TEMP'] = IndexTemplateConfig( + template=template, + # NOTE: Regex magic! Replace all variables. What could possibly go wrong? + values={'\w*': contract}, + ) + self._config.resolve_index_templates() + self._config.pre_initialize() + index_config = self._config.indexes.pop('TEMP') + index_configs = [index_config] + else: + index_configs = list(self._config.indexes.values()) + + for index_config in index_configs: if isinstance(index_config, OperationIndexConfig): for operation_handler_config in index_config.handlers: @@ -157,7 +176,20 @@ async def fetch_schemas(self) -> None: storage_schema = resolve_big_maps(contract_schemas['storageSchema']) write(storage_schema_path, json.dumps(storage_schema, indent=4, sort_keys=True)) - if not isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): + if isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): + pass + elif ( + isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig) + and operation_pattern_config.similar_to + ): + contract_name = operation_pattern_config.similar_to_contract_config.name + for template in self._config.templates: + + # NOTE: We don't know which template will be used in factory handler, so let's try all + with suppress(ConfigurationError): + await self.fetch_schemas(template=template, contract=contract_name) + continue + else: continue parameter_schemas_path = join(contract_schemas_path, 'parameter') @@ -208,7 +240,7 @@ async def fetch_schemas(self) -> None: big_map_value_schema_path = join(big_map_schemas_path, f'{big_map_path}_value.json') write(big_map_value_schema_path, json.dumps(big_map_value_schema, indent=4)) - elif isinstance(index_config, StaticTemplateConfig): + elif isinstance(index_config, IndexTemplateConfig): raise RuntimeError('Config is not pre-initialized') else: diff --git a/src/dipdup/config.py b/src/dipdup/config.py index 815fda2bb..1a91ea9a5 100644 --- a/src/dipdup/config.py +++ b/src/dipdup/config.py @@ -109,7 +109,23 @@ def merge(self, other: Optional['HTTPConfig']) -> None: @dataclass -class ContractConfig: +class NameMixin: + def __post_init_post_parse__(self) -> None: + self._name: Optional[str] = None + + @property + def name(self) -> str: + if self._name is None: + raise RuntimeError('Config is not pre-initialized') + return self._name + + @name.setter + def name(self, name: str) -> None: + self._name = name + + +@dataclass +class ContractConfig(NameMixin): """Contract config :param network: Corresponding network alias, only for sanity checks @@ -135,22 +151,6 @@ def valid_address(cls, v): return v -@dataclass -class NameMixin: - def __post_init_post_parse__(self) -> None: - self._name: Optional[str] = None - - @property - def name(self) -> str: - if self._name is None: - raise RuntimeError('Config is not pre-initialized') - return self._name - - @name.setter - def name(self, name: str) -> None: - self._name = name - - @dataclass class TzktDatasourceConfig(NameMixin): """TzKT datasource config @@ -625,13 +625,13 @@ class BlockIndexConfig(IndexConfig): @dataclass -class StaticTemplateConfig: +class IndexTemplateConfig: kind = 'template' template: str values: Dict[str, str] -IndexConfigT = Union[OperationIndexConfig, BigMapIndexConfig, BlockIndexConfig, StaticTemplateConfig] +IndexConfigT = Union[OperationIndexConfig, BigMapIndexConfig, BlockIndexConfig, IndexTemplateConfig] IndexConfigTemplateT = Union[OperationIndexConfig, BigMapIndexConfig, BlockIndexConfig] HandlerPatternConfigT = Union[OperationHandlerOriginationPatternConfig, OperationHandlerTransactionPatternConfig] @@ -707,7 +707,7 @@ class DipDupConfig: datasources: Dict[str, DatasourceConfigT] contracts: Dict[str, ContractConfig] = Field(default_factory=dict) indexes: Dict[str, IndexConfigT] = Field(default_factory=dict) - templates: Optional[Dict[str, IndexConfigTemplateT]] = None + templates: Dict[str, IndexConfigTemplateT] = Field(default_factory=dict) database: Union[SqliteDatabaseConfig, PostgresDatabaseConfig] = SqliteDatabaseConfig(kind='sqlite') hasura: Optional[HasuraConfig] = None jobs: Optional[Dict[str, JobConfig]] = None @@ -773,10 +773,10 @@ def get_configure_fn(self) -> Type: except (ModuleNotFoundError, AttributeError) as e: raise HandlerImportError(module=module_name, obj=CONFIGURE_HANDLER) from e - def resolve_static_templates(self) -> None: + def resolve_index_templates(self) -> None: _logger.info('Substituting index templates') for index_name, index_config in self.indexes.items(): - if isinstance(index_config, StaticTemplateConfig): + if isinstance(index_config, IndexTemplateConfig): template = self.get_template(index_config.template) raw_template = json.dumps(template, default=pydantic_encoder) for key, value in index_config.values.items(): @@ -840,10 +840,12 @@ def _pre_initialize_index(self, index_name: str, index_config: IndexConfigT) -> self._pre_initialized.append(index_name) def pre_initialize(self) -> None: - for name, config in self.datasources.items(): - config.name = name + for name, contract_config in self.contracts.items(): + contract_config.name = name + for name, datasource_config in self.datasources.items(): + datasource_config.name = name - self.resolve_static_templates() + self.resolve_index_templates() for index_name, index_config in self.indexes.items(): self._pre_initialize_index(index_name, index_config) @@ -941,7 +943,7 @@ def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None if index_name in self._initialized: return - if isinstance(index_config, StaticTemplateConfig): + if isinstance(index_config, IndexTemplateConfig): raise RuntimeError('Config is not pre-initialized') if isinstance(index_config, OperationIndexConfig): diff --git a/src/dipdup/context.py b/src/dipdup/context.py index 81d15a720..79d93ee6b 100644 --- a/src/dipdup/context.py +++ b/src/dipdup/context.py @@ -6,7 +6,7 @@ from tortoise import Tortoise from tortoise.transactions import in_transaction -from dipdup.config import ContractConfig, DipDupConfig, PostgresDatabaseConfig, StaticTemplateConfig +from dipdup.config import ContractConfig, DipDupConfig, IndexTemplateConfig, PostgresDatabaseConfig from dipdup.datasources import DatasourceT from dipdup.exceptions import ContractAlreadyExistsError, IndexAlreadyExistsError from dipdup.utils import FormattedLogger @@ -98,7 +98,7 @@ def add_index(self, name: str, template: str, values: Dict[str, Any]) -> None: if name in self.config.indexes: raise IndexAlreadyExistsError(self, name) self.config.get_template(template) - self.config.indexes[name] = StaticTemplateConfig( + self.config.indexes[name] = IndexTemplateConfig( template=template, values=values, ) diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index 8713c2d59..fc13764ae 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -21,9 +21,9 @@ DatasourceConfigT, DipDupConfig, IndexConfigTemplateT, + IndexTemplateConfig, OperationIndexConfig, PostgresDatabaseConfig, - StaticTemplateConfig, TzktDatasourceConfig, ) from dipdup.context import DipDupContext, RollbackHandlerContext @@ -83,7 +83,7 @@ async def reload_config(self) -> None: self._ctx.config.initialize() for index_config in self._ctx.config.indexes.values(): - if isinstance(index_config, StaticTemplateConfig): + if isinstance(index_config, IndexTemplateConfig): raise RuntimeError await self.add_index(index_config) From 89478fb2c0d23fe03809770541d98c1c9b1eaeee Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Tue, 27 Jul 2021 13:46:09 +0300 Subject: [PATCH 5/8] Rewrite conditions --- src/dipdup/codegen.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index e6e50ee45..d5a406fed 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -176,15 +176,21 @@ async def fetch_schemas(self, template: Optional[str] = None, contract: Optional storage_schema = resolve_big_maps(contract_schemas['storageSchema']) write(storage_schema_path, json.dumps(storage_schema, indent=4, sort_keys=True)) - if isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): + is_transaction = isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig) + is_factory = isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig) and ( + operation_pattern_config.similar_to or operation_pattern_config.source + ) + + if is_transaction: pass - elif ( - isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig) - and operation_pattern_config.similar_to - ): - contract_name = operation_pattern_config.similar_to_contract_config.name - for template in self._config.templates: + elif is_factory: + assert isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig) + if operation_pattern_config.similar_to: + contract_name = operation_pattern_config.similar_to_contract_config.name + elif operation_pattern_config.source: + contract_name = operation_pattern_config.source_contract_config.name + for template in self._config.templates: # NOTE: We don't know which template will be used in factory handler, so let's try all with suppress(ConfigurationError): await self.fetch_schemas(template=template, contract=contract_name) @@ -192,6 +198,7 @@ async def fetch_schemas(self, template: Optional[str] = None, contract: Optional else: continue + assert isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig) parameter_schemas_path = join(contract_schemas_path, 'parameter') entrypoint = cast(str, operation_pattern_config.entrypoint) mkdir_p(parameter_schemas_path) @@ -430,7 +437,10 @@ async def _get_schema( self._schemas[datasource_config] = {} if address not in self._schemas[datasource_config]: if originated: - address = (await datasource.get_originated_contracts(address))[0] + try: + address = (await datasource.get_originated_contracts(address))[0] + except IndexError as e: + raise ConfigurationError(f'Contract `{address}` has no originations') from e self._logger.info('Fetching schemas for contract `%s` (originated from `%s`)', address, contract_config.address) else: self._logger.info('Fetching schemas for contract `%s`', address) From cbd113c6fbb5ad9006e0e226c393ee598ba4ce75 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Thu, 29 Jul 2021 09:36:38 +0300 Subject: [PATCH 6/8] Revert template schema fetching --- src/dipdup/codegen.py | 47 ++++--------------------------------------- 1 file changed, 4 insertions(+), 43 deletions(-) diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index d5a406fed..315e1cdf3 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -3,11 +3,10 @@ import os import re import subprocess -from contextlib import suppress from copy import copy from os.path import basename, dirname, exists, join, relpath, splitext from shutil import rmtree -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, cast from jinja2 import Template @@ -19,7 +18,6 @@ ContractConfig, DatasourceConfigT, DipDupConfig, - IndexConfigT, IndexTemplateConfig, OperationHandlerOriginationPatternConfig, OperationHandlerTransactionPatternConfig, @@ -123,30 +121,13 @@ async def create_package(self) -> None: graphql_path = join(self._config.package_path, 'graphql') touch(join(graphql_path, '.keep')) - async def fetch_schemas(self, template: Optional[str] = None, contract: Optional[str] = None) -> None: + async def fetch_schemas(self) -> None: """Fetch JSONSchemas for all contracts used in config""" self._logger.info('Creating `schemas` package') schemas_path = join(self._config.package_path, 'schemas') mkdir_p(schemas_path) - index_configs: List[IndexConfigT] - if template: - if not contract: - raise RuntimeError('Both `template` and `contract` arguments are required') - # FIXME: Implement classmethods to avoid adding temporary index - self._config.indexes['TEMP'] = IndexTemplateConfig( - template=template, - # NOTE: Regex magic! Replace all variables. What could possibly go wrong? - values={'\w*': contract}, - ) - self._config.resolve_index_templates() - self._config.pre_initialize() - index_config = self._config.indexes.pop('TEMP') - index_configs = [index_config] - else: - index_configs = list(self._config.indexes.values()) - - for index_config in index_configs: + for index_config in self._config.indexes.values(): if isinstance(index_config, OperationIndexConfig): for operation_handler_config in index_config.handlers: @@ -176,29 +157,9 @@ async def fetch_schemas(self, template: Optional[str] = None, contract: Optional storage_schema = resolve_big_maps(contract_schemas['storageSchema']) write(storage_schema_path, json.dumps(storage_schema, indent=4, sort_keys=True)) - is_transaction = isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig) - is_factory = isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig) and ( - operation_pattern_config.similar_to or operation_pattern_config.source - ) - - if is_transaction: - pass - elif is_factory: - assert isinstance(operation_pattern_config, OperationHandlerOriginationPatternConfig) - if operation_pattern_config.similar_to: - contract_name = operation_pattern_config.similar_to_contract_config.name - elif operation_pattern_config.source: - contract_name = operation_pattern_config.source_contract_config.name - - for template in self._config.templates: - # NOTE: We don't know which template will be used in factory handler, so let's try all - with suppress(ConfigurationError): - await self.fetch_schemas(template=template, contract=contract_name) - continue - else: + if not isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): continue - assert isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig) parameter_schemas_path = join(contract_schemas_path, 'parameter') entrypoint = cast(str, operation_pattern_config.entrypoint) mkdir_p(parameter_schemas_path) From 9b6ebdcce561fd51dca8c40fed235b3ae16ed2f9 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Thu, 29 Jul 2021 10:35:42 +0300 Subject: [PATCH 7/8] message --- src/dipdup/codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index 315e1cdf3..51c9e2452 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -401,7 +401,7 @@ async def _get_schema( try: address = (await datasource.get_originated_contracts(address))[0] except IndexError as e: - raise ConfigurationError(f'Contract `{address}` has no originations') from e + raise ConfigurationError(f'No contracts were originated from `{address}`') from e self._logger.info('Fetching schemas for contract `%s` (originated from `%s`)', address, contract_config.address) else: self._logger.info('Fetching schemas for contract `%s`', address) From 5df0951195c59336d9389be6211ddc22156b0822 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Thu, 29 Jul 2021 10:51:37 +0300 Subject: [PATCH 8/8] Suppress empty SQL --- src/dipdup/dipdup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index fc13764ae..4008e564c 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -1,7 +1,7 @@ import asyncio import hashlib import logging -from contextlib import AsyncExitStack, asynccontextmanager +from contextlib import AsyncExitStack, asynccontextmanager, suppress from os import listdir from os.path import join from typing import Dict, List, Optional, cast @@ -339,7 +339,8 @@ async def _execute_sql_scripts(self, reindex: bool) -> None: for file in iter_files(sql_path, '.sql'): self._logger.info('Executing `%s`', file.name) sql = file.read() - await get_connection(None).execute_script(sql) + with suppress(AttributeError): + await get_connection(None).execute_script(sql) def _finish_migration(self, version: str) -> None: self._logger.warning('==================== WARNING =====================')