Merge pull request #70 from ncloudioj/master

Add support for multiple databases migration

MANIFEST.in

@@ -1,2 +1,3 @@
-include README.md LICENSE flask_migrate/templates/flask/* tests/*
+include README.md LICENSE flask_migrate/templates/flask/* \
+flask_migrate/templates/flask-multidb/* tests/*
 

docs/index.rst

@@ -82,8 +82,10 @@ The application will now have a ``db`` command line option with several sub-comm
 - ``manage.py db --help``
     Shows a list of available commands.
 
-- ``manage.py db init``
-    Initializes migration support for the application.
+- ``manage.py db init [--multidb]``
+    Initializes migration support for the application. Turning on option ``--multidb`` will create multiple databases templates for alembic. This feature could be used with `Flask-SQLAlchemy Binds<https://pythonhosted.org/Flask-SQLAlchemy/binds.html>`. Note that you do *NOT* need this option for other commands, e.g. migrate, upgrade, downgrade, etc. Two more steps are needed once you get the alembic template files:
+    1. Add all database names to the filed ``databases`` in ``alembic.ini``. The ``SQLALCHEMY_DATABASE_URI`` is by default already set as "primary" (you can customize it, but make sure it also gets updated in the ``env.py``), all keys in the ``SQLALCHEMY_BINDS`` should be append to ``database`` filed as a comma seperated string.
+    2. Set ``target_metadata`` in ``env.py``, each database should have a ``db`` object, which, in turn, has all the table information in the ``metadata``. See more detail in the template comment.
 
 - ``manage.py db revision [--message MESSAGE] [--autogenerate] [--sql] [--head HEAD] [--splice] [--branch-label BRANCH_LABEL] [--version-path VERSION_PATH] [--rev-id REV_ID]``
     Creates an empty revision script. The script needs to be edited manually with the upgrade and downgrade changes. See `Alembic's documentation <https://alembic.readthedocs.org/en/latest/index.html>`_ for instructions on how to write migration scripts. An optional migration message can be included.
@@ -130,7 +132,7 @@ API Reference
 
 The commands exposed by Flask-Migrate's interface to Flask-Script can also be accessed programmatically by importing the functions from module ``flask.ext.migrate``. The available functions are:
 
-- ``init(directory='migrations')``
+- ``init(directory='migrations', multidb=False)``
     Initializes migration support for the application.
 
 - ``revision(directory='migrations', message=None, autogenerate=False, sql=False, head='head', splice=False, branch_label=None, version_path=None, rev_id=None)``

flask_migrate/__init__.py

@@ -59,14 +59,21 @@ def _get_config(directory, x_arg=None):
 @MigrateCommand.option('-d', '--directory', dest='directory', default=None,
                        help=("migration script directory (default is "
                              "'migrations')"))
-def init(directory=None):
+@MigrateCommand.option('-m', '--multidb', dest='multidb', action='store_true',
+                       default=False,
+                       help=("multiple databases migraton (default is "
+                             "False)"))
+def init(directory=None, multidb=False):
     """Generates a new migration"""
     if directory is None:
         directory = current_app.extensions['migrate'].directory
     config = Config()
     config.set_main_option('script_location', directory)
     config.config_file_name = os.path.join(directory, 'alembic.ini')
-    command.init(config, directory, 'flask')
+    if multidb:
+        command.init(config, directory, 'flask-multidb')
+    else:
+        command.init(config, directory, 'flask')
 
 
 @MigrateCommand.option('--rev-id', dest='rev_id', default=None,

flask_migrate/templates/flask-multidb/README

@@ -0,0 +1 @@
+Generic single-database configuration.

flask_migrate/templates/flask-multidb/alembic.ini.mako

@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# comma seperated database names, the default database
+# (FLASK_SQLALCHEMY_URL) is primary, other names must be
+# the same as names in SQLALCHEMY_BINDS
+# e.g. database = primary, db1, db2, ...
+databases = primary
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

flask_migrate/templates/flask-multidb/env.py

@@ -0,0 +1,142 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+import logging
+import re
+
+USE_TWOPHASE = False
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# gather section names referring to different
+# databases.
+db_names = config.get_main_option('databases')
+
+# gather the database engine's information
+from flask import current_app
+config.set_main_option('sqlalchemy.url',
+                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+context.config.set_section_option("primary", "sqlalchemy.url",
+                                  current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+for engine, url in current_app.config.get("SQLALCHEMY_BINDS").items():
+    context.config.set_section_option(engine, "sqlalchemy.url", url)
+
+# add your model's MetaData objects here
+# for 'autogenerate' support.  These must be set
+# up to hold just those tables targeting a
+# particular database. table.tometadata() may be
+# helpful here in case a "copy" of
+# a MetaData is needed.
+# from myapp import mymodel
+# target_metadata = {
+#       'engine1':mymodel.metadata1,
+#       'engine2':mymodel.metadata2
+#}
+target_metadata = {
+}
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    # for the --sql use case, run migrations for each URL into
+    # individual files.
+
+    engines = {}
+    for name in re.split(r',\s*', db_names):
+        engines[name] = rec = {}
+        rec['url'] = context.config.get_section_option(name,
+                                                       "sqlalchemy.url")
+
+    for name, rec in engines.items():
+        logger.info("Migrating database %s" % name)
+        file_ = "%s.sql" % name
+        logger.info("Writing output to %s" % file_)
+        with open(file_, 'w') as buffer:
+            context.configure(url=rec['url'], output_buffer=buffer,
+                              target_metadata=target_metadata.get(name),
+                              literal_binds=True)
+            with context.begin_transaction():
+                context.run_migrations(engine_name=name)
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    # for the direct-to-DB use case, start a transaction on all
+    # engines, then run all migrations, then commit all transactions.
+
+    engines = {}
+    for name in re.split(r',\s*', db_names):
+        engines[name] = rec = {}
+        rec['engine'] = engine_from_config(
+            context.config.get_section(name),
+            prefix='sqlalchemy.',
+            poolclass=pool.NullPool)
+
+    for name, rec in engines.items():
+        engine = rec['engine']
+        rec['connection'] = conn = engine.connect()
+
+        if USE_TWOPHASE:
+            rec['transaction'] = conn.begin_twophase()
+        else:
+            rec['transaction'] = conn.begin()
+
+    try:
+        for name, rec in engines.items():
+            logger.info("Migrating database %s" % name)
+            context.configure(
+                connection=rec['connection'],
+                upgrade_token="%s_upgrades" % name,
+                downgrade_token="%s_downgrades" % name,
+                target_metadata=target_metadata.get(name)
+            )
+            context.run_migrations(engine_name=name)
+
+        if USE_TWOPHASE:
+            for rec in engines.values():
+                rec['transaction'].prepare()
+
+        for rec in engines.values():
+            rec['transaction'].commit()
+    except:
+        for rec in engines.values():
+            rec['transaction'].rollback()
+        raise
+    finally:
+        for rec in engines.values():
+            rec['connection'].close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

flask_migrate/templates/flask-multidb/script.py.mako

@@ -0,0 +1,45 @@
+<%!
+import re
+
+%>"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade(engine_name):
+    globals()["upgrade_%s" % engine_name]()
+
+
+def downgrade(engine_name):
+    globals()["downgrade_%s" % engine_name]()
+
+<%
+    db_names = config.get_main_option("databases")
+%>
+
+## generate an "upgrade_<xyz>() / downgrade_<xyz>()" function
+## for each database name in the ini file.
+
+% for db_name in re.split(r',\s*', db_names):
+
+def upgrade_${db_name}():
+    ${context.get("%s_upgrades" % db_name, "pass")}
+
+
+def downgrade_${db_name}():
+    ${context.get("%s_downgrades" % db_name, "pass")}
+
+% endfor

tests/multidb/alembic.ini

@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# comma seperated database names, the default database
+# (FLASK_SQLALCHEMY_URL) is primary, other names must be
+# the same as names in SQLALCHEMY_BINDS
+# e.g. database = primary, db1, db2, ...
+databases = primary, db1
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

tests/multidb/app_multidb.py

@@ -0,0 +1,34 @@
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+from flask_script import Manager
+from flask_migrate import Migrate, MigrateCommand
+
+app = Flask(__name__)
+app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///app.db'
+app.config['SQLALCHEMY_BINDS'] = {
+    "db1": "sqlite:///app1.db",
+}
+
+db = SQLAlchemy(app)
+migrate = Migrate(app, db)
+db1 = SQLAlchemy(app)
+
+manager = Manager(app)
+manager.add_command('db', MigrateCommand)
+
+metadata = db.metadata
+metadata1 = db1.metadata
+
+
+class User(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(128))
+
+
+class Group(db1.Model):
+    id = db1.Column(db1.Integer, primary_key=True)
+    name = db1.Column(db1.String(128))
+
+
+if __name__ == '__main__':
+    manager.run()