Add "remove" BigMapAction, rename others (#79)

src/demo_hic_et_nunc/dipdup.yml

@@ -1,15 +1,15 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_hic_et_nunc
 
 database:
   kind: sqlite
   path: hic_et_nunc.sqlite3
 
 contracts:
-  HEN_objkts: 
+  HEN_objkts:
     address: ${HEN_OBJKTS:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton}
     typename: hen_objkts
-  HEN_minter: 
+  HEN_minter:
     address: ${HEN_MINTER:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9}
     typename: hen_minter
 

src/demo_quipuswap/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_quipuswap
 
 database:

src/demo_registrydao/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_registrydao
 
 database:

src/demo_tezos_domains/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tezos_domains
 
 database:

src/demo_tezos_domains_big_map/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tezos_domains_big_map
 
 database:

src/demo_tezos_domains_big_map/handlers/on_update_expiry_map.py

@@ -2,15 +2,16 @@
 from demo_tezos_domains_big_map.types.name_registry.big_map.store_expiry_map_key import StoreExpiryMapKey
 from demo_tezos_domains_big_map.types.name_registry.big_map.store_expiry_map_value import StoreExpiryMapValue
 from dipdup.context import HandlerContext
-from dipdup.models import BigMapAction, BigMapDiff
+from dipdup.models import BigMapDiff
 
 
 async def on_update_expiry_map(
     ctx: HandlerContext,
     store_expiry_map: BigMapDiff[StoreExpiryMapKey, StoreExpiryMapValue],
 ) -> None:
-    if store_expiry_map.action == BigMapAction.REMOVE:
+    if not store_expiry_map.action.has_value:
         return
+    assert store_expiry_map.key
     assert store_expiry_map.value
 
     expiry = store_expiry_map.value.__root__

src/demo_tezos_domains_big_map/handlers/on_update_records.py

@@ -2,15 +2,16 @@
 from demo_tezos_domains_big_map.types.name_registry.big_map.store_records_key import StoreRecordsKey
 from demo_tezos_domains_big_map.types.name_registry.big_map.store_records_value import StoreRecordsValue
 from dipdup.context import HandlerContext
-from dipdup.models import BigMapAction, BigMapDiff
+from dipdup.models import BigMapDiff
 
 
 async def on_update_records(
     ctx: HandlerContext,
     store_records: BigMapDiff[StoreRecordsKey, StoreRecordsValue],
 ) -> None:
-    if store_records.action == BigMapAction.REMOVE:
+    if not store_records.action.has_value:
         return
+    assert store_records.key
     assert store_records.value
 
     record_name = bytes.fromhex(store_records.key.__root__).decode()

src/demo_tzbtc/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tzbtc
 
 database:
@@ -29,4 +29,4 @@ indexes:
       - callback: on_mint
         pattern:
           - destination: tzbtc_mainnet
-            entrypoint: mint
+            entrypoint: mint

src/demo_tzcolors/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tzcolors
 
 database:
@@ -49,4 +49,4 @@ indexes:
     values:
       datasource: tzkt
       minter: tzcolors_minter
-      auction: tzcolors_auction
+      auction: tzcolors_auction

src/dipdup/__init__.py

@@ -1,2 +1,2 @@
 __version__ = '1.1.2'
-__spec_version__ = '1.0'
+__spec_version__ = '1.1'

src/dipdup/cli.py

@@ -15,12 +15,14 @@
 from dipdup import __spec_version__, __version__
 from dipdup.config import DipDupConfig, LoggingConfig
 from dipdup.dipdup import DipDup
+from dipdup.exceptions import ConfigurationError
 
 _logger = logging.getLogger(__name__)
 
 spec_version_to_version = {
-    '0.1': 'dipdup <0.4.3',
-    '1.0': 'dipdup ^1.0.0',
+    '0.1': 'dipdup v0.4.3 and below',
+    '1.0': 'dipdup v1.0.0 - v1.1.2',
+    '1.1': 'dipdup v1.2.0 and above',
 }
 
 migration_required_message = """
@@ -79,6 +81,8 @@ async def cli(ctx, config: List[str], logging_config: str):
     _logging_config.apply()
 
     _config = DipDupConfig.load(config)
+    if _config.spec_version not in spec_version_to_version:
+        raise ConfigurationError('Unknown `spec_version`')
     if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate':
         migration_required(_config.spec_version, __spec_version__)
 
@@ -121,16 +125,27 @@ async def init(ctx):
 @click.pass_context
 @click_async
 async def migrate(ctx):
+    def _bump_spec_version(spec_version: str):
+        for config_path in ctx.obj.config_paths:
+            for line in fileinput.input(config_path, inplace=True):
+                if 'spec_version' in line:
+                    print(f'spec_version: {spec_version}')
+                else:
+                    print(line.rstrip())
+
     config: DipDupConfig = ctx.obj.config
     config.pre_initialize()
-    await DipDup(config).migrate()
-
-    for config_path in ctx.obj.config_paths:
-        for line in fileinput.input(config_path, inplace=True):
-            if 'spec_version' in line:
-                print(f'spec_version: {__spec_version__}')
-            else:
-                print(line.rstrip())
+
+    if config.spec_version == __spec_version__:
+        _logger.error('Project is already at latest version')
+    elif config.spec_version == '0.1':
+        await DipDup(config).migrate_to_v10()
+        _bump_spec_version('1.0')
+    elif config.spec_version == '1.0':
+        await DipDup(config).migrate_to_v11()
+        _bump_spec_version('1.1')
+    else:
+        raise ConfigurationError('Unknown `spec_version`')
 
 
 @cli.command(help='Clear development request cache')

src/dipdup/codegen.py

@@ -371,7 +371,7 @@ async def _get_schema(
             self._schemas[datasource_config][address] = address_schemas_json
         return self._schemas[datasource_config][address]
 
-    async def migrate_user_handlers_to_v1(self) -> None:
+    async def migrate_user_handlers_to_v10(self) -> None:
         remove_lines = [
             'from dipdup.models import',
             'from dipdup.context import',
@@ -407,3 +407,26 @@ async def migrate_user_handlers_to_v1(self) -> None:
                         newfile.append(line)
                 with open(path, 'w') as file:
                     file.write('\n'.join(newfile))
+
+    async def migrate_user_handlers_to_v11(self) -> None:
+        replace_table = {
+            'BigMapAction.ADD': 'BigMapAction.ADD_KEY',
+            'BigMapAction.UPDATE': 'BigMapAction.UPDATE_KEY',
+            'BigMapAction.REMOVE': 'BigMapAction.REMOVE_KEY',
+        }
+        handlers_path = join(self._config.package_path, 'handlers')
+
+        for root, _, files in os.walk(handlers_path):
+            for filename in files:
+                if filename == '__init__.py' or not filename.endswith('.py'):
+                    continue
+                path = join(root, filename)
+                newfile = []
+                with open(path) as file:
+                    for line in file.read().split('\n'):
+                        # Replace by table
+                        for from_, to in replace_table.items():
+                            line = line.replace(from_, to)
+                        newfile.append(line)
+                with open(path, 'w') as file:
+                    file.write('\n'.join(newfile))

src/dipdup/dipdup.py

@@ -34,7 +34,7 @@
 from dipdup.datasources.coinbase.datasource import CoinbaseDatasource
 from dipdup.datasources.datasource import IndexDatasource
 from dipdup.datasources.tzkt.datasource import TzktDatasource
-from dipdup.exceptions import ConfigurationError, HandlerImportError
+from dipdup.exceptions import ConfigurationError
 from dipdup.hasura import configure_hasura
 from dipdup.index import BigMapIndex, Index, OperationIndex
 from dipdup.models import BigMapData, IndexType, OperationData, State
@@ -205,22 +205,20 @@ async def run(self, reindex: bool, oneshot: bool) -> None:
                 with suppress(AttributeError, SchedulerNotRunningError):
                     self._scheduler.shutdown(wait=True)
 
-    async def migrate(self) -> None:
+    async def migrate_to_v10(self) -> None:
         codegen = DipDupCodeGenerator(self._config, self._datasources_by_config)
         await codegen.generate_default_handlers(recreate=True)
-        await codegen.migrate_user_handlers_to_v1()
-        self._logger.warning('==================== WARNING =====================')
-        self._logger.warning('Your handlers have just been migrated to v1.0.0 format.')
-        self._logger.warning('Review and commit changes before proceeding.')
-        self._logger.warning('==================== WARNING =====================')
-        quit()
+        await codegen.migrate_user_handlers_to_v10()
+        self._finish_migration('1.0')
+
+    async def migrate_to_v11(self) -> None:
+        codegen = DipDupCodeGenerator(self._config, self._datasources_by_config)
+        await codegen.migrate_user_handlers_to_v11()
+        self._finish_migration('1.1')
 
     async def _configure(self) -> None:
         """Run user-defined initial configuration handler"""
-        try:
-            configure_fn = self._config.get_configure_fn()
-        except HandlerImportError:
-            await self.migrate()
+        configure_fn = self._config.get_configure_fn()
         await configure_fn(self._ctx)
         self._config.initialize()
 
@@ -313,3 +311,9 @@ async def _execute_sql_scripts(self, reindex: bool) -> None:
 
             self._logger.info('Executing `%s`', filename)
             await get_connection(None).execute_script(sql)
+
+    def _finish_migration(self, version: str) -> None:
+        self._logger.warning('==================== WARNING =====================')
+        self._logger.warning('Your project has been migrated to spec version %s.', version)
+        self._logger.warning('Review and commit changes before proceeding.')
+        self._logger.warning('==================== WARNING =====================')

src/dipdup/index.py

@@ -18,7 +18,7 @@
 )
 from dipdup.context import DipDupContext, HandlerContext
 from dipdup.datasources.tzkt.datasource import BigMapFetcher, OperationFetcher, TzktDatasource
-from dipdup.models import BigMapAction, BigMapData, BigMapDiff, OperationData, Origination, State, TemporaryState, Transaction
+from dipdup.models import BigMapData, BigMapDiff, OperationData, Origination, State, TemporaryState, Transaction
 from dipdup.utils import FormattedLogger, in_global_transaction
 
 OperationGroup = namedtuple('OperationGroup', ('hash', 'counter'))
@@ -400,17 +400,18 @@ async def _on_match(
         matched_big_map: BigMapData,
     ) -> None:
         """Prepare handler arguments, parse key and value. Schedule callback in executor."""
-        if matched_big_map.action == BigMapAction.ALLOCATE:
-            return
-
-        key_type = handler_config.key_type_cls
-        key = key_type.parse_obj(matched_big_map.key)
 
-        if matched_big_map.action == BigMapAction.REMOVE:
-            value = None
+        if matched_big_map.action.has_key:
+            key_type = handler_config.key_type_cls
+            key = key_type.parse_obj(matched_big_map.key)
         else:
+            key = None
+
+        if matched_big_map.action.has_value:
             value_type = handler_config.value_type_cls
             value = value_type.parse_obj(matched_big_map.value)
+        else:
+            value = None
 
         big_map_context = BigMapDiff(  # type: ignore
             data=matched_big_map,

src/dipdup/models.py

@@ -174,9 +174,18 @@ class BigMapAction(Enum):
     """Mapping for action in TzKT response"""
 
     ALLOCATE = 'allocate'
-    ADD = 'add_key'
-    UPDATE = 'update_key'
-    REMOVE = 'remove_key'
+    ADD_KEY = 'add_key'
+    UPDATE_KEY = 'update_key'
+    REMOVE_KEY = 'remove_key'
+    REMOVE = 'remove'
+
+    @property
+    def has_key(self) -> bool:
+        return self in (BigMapAction.ADD_KEY, BigMapAction.UPDATE_KEY, BigMapAction.REMOVE_KEY)
+
+    @property
+    def has_value(self) -> bool:
+        return self in (BigMapAction.ADD_KEY, BigMapAction.UPDATE_KEY)
 
 
 @dataclass
@@ -201,5 +210,5 @@ class BigMapDiff(Generic[KeyType, ValueType]):
 
     action: BigMapAction
     data: BigMapData
-    key: KeyType
+    key: Optional[KeyType]
     value: Optional[ValueType]

tests/integration_tests/hic_et_nunc.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_hic_et_nunc
 
 database:

tests/integration_tests/hic_et_nunc_job.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_hic_et_nunc
 
 database:

tests/integration_tests/quipuswap.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_quipuswap
 
 database:

tests/integration_tests/registrydao.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_registrydao
 
 database:

tests/integration_tests/tezos_domains.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tezos_domains
 
 database:

tests/integration_tests/tezos_domains_big_map.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tezos_domains_big_map
 
 database:

tests/integration_tests/tzcolors.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_tzcolors
 
 database:

tests/test_dipdup/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 1.0
+spec_version: 1.1
 package: demo_hic_et_nunc
 
 database: