Hasura v2: user limits and camelcase (#80)

docker-compose.yml

@@ -72,7 +72,7 @@ services:
       retries: 5
 
   hasura:
-    image: hasura/graphql-engine:v1.3.3
+    image: hasura/graphql-engine:v2.0.1
     ports:
       - 127.0.0.1:8080:8080
     depends_on:

pyproject.toml

@@ -34,6 +34,7 @@ click = "^8.0.1"
 pyee = "^8.1.0"
 APScheduler = "^3.7.0"
 sentry-sdk = "^1.1.0"
+pyhumps = "^3.0.2"
 aiolimiter = "^1.0.0-beta.1"
 
 [tool.poetry.dev-dependencies]

scripts/init_demos.sh

@@ -0,0 +1,5 @@
+#$/bin/bash
+for name in `ls src | grep demo`
+do
+    dipdup -c src/$name/dipdup.yml init
+done

scripts/migrate_demos.sh

@@ -0,0 +1,5 @@
+#$/bin/bash
+for name in `ls src | grep demo`
+do
+    dipdup -c src/$name/dipdup.yml migrate
+done

src/demo_hic_et_nunc/dipdup-local.yml

@@ -0,0 +1,13 @@
+database:
+  kind: postgres
+  host: 127.0.0.1
+  port: 6423
+  user: ${POSTGRES_USER:-dipdup}
+  password: ${POSTGRES_PASSWORD:-changeme}
+  database: ${POSTGRES_DB:-dipdup}
+  schema_name: hic_et_nunc
+
+hasura:
+  url: http://127.0.0.1:8080
+  admin_secret: ${ADMIN_SECRET:-changeme}
+  camel_case: True

src/demo_hic_et_nunc/graphql/token.graphql

@@ -0,0 +1,9 @@
+query tokens($id: bigint) {
+  hicEtNuncToken(where: {id: {_eq: $id}}) {
+    creatorId
+    id
+    level
+    supply
+    timestamp
+  }
+}

src/dipdup/cli.py

@@ -5,17 +5,19 @@
 from dataclasses import dataclass
 from functools import wraps
 from os.path import dirname, join
-from typing import List, NoReturn
+from typing import List, NoReturn, cast
 
 import click
 import sentry_sdk
 from fcache.cache import FileCache  # type: ignore
 from sentry_sdk.integrations.aiohttp import AioHttpIntegration
 
 from dipdup import __spec_version__, __version__
-from dipdup.config import DipDupConfig, LoggingConfig
+from dipdup.config import DipDupConfig, LoggingConfig, PostgresDatabaseConfig
 from dipdup.dipdup import DipDup
 from dipdup.exceptions import ConfigurationError
+from dipdup.hasura import HasuraManager
+from dipdup.utils import tortoise_wrapper
 
 _logger = logging.getLogger(__name__)
 
@@ -153,3 +155,43 @@ def _bump_spec_version(spec_version: str):
 @click_async
 async def clear_cache(ctx):
     FileCache('dipdup', flag='cs').clear()
+
+
+@cli.command(help='Configure Hasura GraphQL Engine')
+@click.option('--reset', is_flag=True, help='Reset metadata before configuring')
+@click.pass_context
+@click_async
+async def configure_hasura(ctx, reset: bool):
+    config: DipDupConfig = ctx.obj.config
+    url = config.database.connection_string
+    models = f'{config.package}.models'
+    if not config.hasura:
+        _logger.error('`hasura` config section is empty')
+        return
+    hasura = HasuraManager(config.package, config.hasura, cast(PostgresDatabaseConfig, config.database))
+
+    async with tortoise_wrapper(url, models):
+        try:
+            await hasura.configure(reset)
+        finally:
+            await hasura.close_session()
+
+
+@cli.command(help='Configure Hasura GraphQL Engine')
+@click.option('--reset', is_flag=True, help='Reset metadata before configuring')
+@click.pass_context
+@click_async
+async def cache(ctx, reset: bool):
+    config: DipDupConfig = ctx.obj.config
+    url = config.database.connection_string
+    models = f'{config.package}.models'
+    if not config.hasura:
+        _logger.error('`hasura` config section is empty')
+        return
+    hasura = HasuraManager(config.package, config.hasura, cast(PostgresDatabaseConfig, config.database))
+
+    async with tortoise_wrapper(url, models):
+        try:
+            await hasura.configure(reset)
+        finally:
+            await hasura.close_session()

src/dipdup/codegen.py

@@ -95,9 +95,20 @@ async def create_package(self) -> None:
         sql_on_restart_path = join(sql_path, 'on_restart')
         with suppress(FileExistsError):
             mkdir(sql_on_restart_path)
+        with open(join(sql_on_restart_path, '.keep'), 'w'):
+            pass
         sql_on_reindex_path = join(sql_path, 'on_reindex')
         with suppress(FileExistsError):
             mkdir(sql_on_reindex_path)
+        with open(join(sql_on_reindex_path, '.keep'), 'w'):
+            pass
+
+        self._logger.info('Creating `%s/graphql` directory', self._config.package)
+        graphql_path = join(self._config.package_path, 'graphql')
+        with suppress(FileExistsError):
+            mkdir(graphql_path)
+        with open(join(graphql_path, '.keep'), 'w'):
+            pass
 
     async def fetch_schemas(self) -> None:
         """Fetch JSONSchemas for all contracts used in config"""

src/dipdup/config.py

@@ -89,6 +89,15 @@ def valid_immune_tables(cls, v):
         return v
 
 
+@dataclass
+class HTTPConfig:
+    cache: bool = True
+    retry_count: int = DEFAULT_RETRY_COUNT
+    retry_sleep: int = DEFAULT_RETRY_SLEEP
+    ratelimit_rate: Optional[int] = None
+    ratelimit_period: Optional[int] = None
+
+
 @dataclass
 class ContractConfig:
     """Contract config
@@ -141,10 +150,7 @@ class TzktDatasourceConfig(NameMixin):
 
     kind: Literal['tzkt']
     url: str
-
-    cache: Optional[bool] = None
-    retry_count: int = DEFAULT_RETRY_COUNT
-    retry_sleep: int = DEFAULT_RETRY_SLEEP
+    http: Optional[HTTPConfig] = None
 
     def __hash__(self):
         return hash(self.url)
@@ -167,10 +173,7 @@ class BcdDatasourceConfig(NameMixin):
     kind: Literal['bcd']
     url: str
     network: str
-
-    cache: Optional[bool] = None
-    retry_count: int = DEFAULT_RETRY_COUNT
-    retry_sleep: int = DEFAULT_RETRY_SLEEP
+    http: Optional[HTTPConfig] = None
 
     def __hash__(self):
         return hash(self.url + self.network)
@@ -189,10 +192,7 @@ class CoinbaseDatasourceConfig(NameMixin):
     api_key: Optional[str] = None
     secret_key: Optional[str] = None
     passphrase: Optional[str] = None
-
-    cache: Optional[bool] = None
-    retry_count: int = DEFAULT_RETRY_COUNT
-    retry_sleep: int = DEFAULT_RETRY_SLEEP
+    http: Optional[HTTPConfig] = None
 
     def __hash__(self):
         return hash(self.kind)
@@ -630,13 +630,31 @@ class StaticTemplateConfig:
 class HasuraConfig:
     url: str
     admin_secret: Optional[str] = None
+    source: str = 'default'
+    select_limit: int = 100
+    allow_aggregations: bool = True
+    camel_case: bool = False
+    connection_timeout: int = 5
+    rest: bool = True
+    http: Optional[HTTPConfig] = None
 
     @validator('url', allow_reuse=True)
     def valid_url(cls, v):
         parsed_url = urlparse(v)
         if not (parsed_url.scheme and parsed_url.netloc):
             raise ConfigurationError(f'`{v}` is not a valid Hasura URL')
-        return v
+        return v.rstrip('/')
+
+    @validator('source', allow_reuse=True)
+    def valid_source(cls, v):
+        if v != 'default':
+            raise NotImplementedError('Multiple Hasura sources are not supported at the moment')
+
+    @property
+    def headers(self) -> Dict[str, str]:
+        if self.admin_secret:
+            return {'X-Hasura-Admin-Secret': self.admin_secret}
+        return {}
 
 
 @dataclass