Skip to content

Commit

Permalink
add new migrations system (#10312)
Browse files Browse the repository at this point in the history
  • Loading branch information
@FredKSchott
FredKSchott authored Mar 4, 2024
1 parent 718eed7 commit 93ec9e2
Show file tree
Hide file tree
Showing 15 changed files with 188 additions and 603 deletions.
5 changes: 5 additions & 0 deletions .changeset/good-maps-deny.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@astrojs/db": minor
---

Revamp migrations system
56 changes: 0 additions & 56 deletions packages/db/src/core/cli/commands/gen/index.ts
View file

This file was deleted.

175 changes: 28 additions & 147 deletions packages/db/src/core/cli/commands/push/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,16 @@
import type { AstroConfig } from 'astro';
import { red } from 'kleur/colors';
import prompts from 'prompts';
import type { Arguments } from 'yargs-parser';
import { MISSING_SESSION_ID_ERROR } from '../../../errors.js';
import { getManagedAppTokenOrExit } from '../../../tokens.js';
import { type DBConfig, type DBSnapshot } from '../../../types.js';
import { getMigrationsDirectoryUrl, getRemoteDatabaseUrl } from '../../../utils.js';
import { getMigrationQueries } from '../../migration-queries.js';
import { getRemoteDatabaseUrl } from '../../../utils.js';
import {
INITIAL_SNAPSHOT,
MIGRATIONS_NOT_INITIALIZED,
MIGRATIONS_UP_TO_DATE,
MIGRATION_NEEDED,
getMigrationQueries,
createCurrentSnapshot,
createEmptySnapshot,
getMigrationStatus,
getMigrations,
loadInitialSnapshot,
loadMigration,
} from '../../migrations.js';
getProductionCurrentSnapshot,
} from '../../migration-queries.js';

export async function cmd({
astroConfig,
dbConfig,
flags,
}: {
Expand All @@ -29,49 +19,31 @@ export async function cmd({
flags: Arguments;
}) {
const isDryRun = flags.dryRun;
const isForceReset = flags.forceReset;
const appToken = await getManagedAppTokenOrExit(flags.token);
const migration = await getMigrationStatus({ dbConfig, root: astroConfig.root });
if (migration.state === 'no-migrations-found') {
console.log(MIGRATIONS_NOT_INITIALIZED);
process.exit(1);
} else if (migration.state === 'ahead') {
console.log(MIGRATION_NEEDED);
process.exit(1);
}
const migrationsDir = getMigrationsDirectoryUrl(astroConfig.root);
const productionSnapshot = await getProductionCurrentSnapshot({ appToken: appToken.token });
const currentSnapshot = createCurrentSnapshot(dbConfig);
const isFromScratch = isForceReset || JSON.stringify(productionSnapshot) === '{}';
const { queries: migrationQueries } = await getMigrationQueries({
oldSnapshot: isFromScratch ? createEmptySnapshot() : productionSnapshot,
newSnapshot: currentSnapshot,
});

// get all migrations from the filesystem
const allLocalMigrations = await getMigrations(migrationsDir);
let missingMigrations: string[] = [];
try {
const { data } = await prepareMigrateQuery({
migrations: allLocalMigrations,
appToken: appToken.token,
});
missingMigrations = data;
} catch (error) {
if (error instanceof Error) {
if (error.message.startsWith('{')) {
const { error: { code } = { code: '' } } = JSON.parse(error.message);
if (code === 'TOKEN_UNAUTHORIZED') {
console.error(MISSING_SESSION_ID_ERROR);
}
}
}
console.error(error);
process.exit(1);
// // push the database schema
if (migrationQueries.length === 0) {
console.log('Database schema is up to date.');
} else {
console.log(`Database schema is out of date.`);
}
// push the database schema
if (missingMigrations.length === 0) {
console.log(MIGRATIONS_UP_TO_DATE);
if (isDryRun) {
console.log('Statements:', JSON.stringify(migrationQueries, undefined, 2));
} else {
console.log(`Pushing ${missingMigrations.length} migrations...`);
console.log(`Pushing database schema updates...`);
await pushSchema({
migrations: missingMigrations,
migrationsDir,
statements: migrationQueries,
appToken: appToken.token,
isDryRun,
currentSnapshot: migration.currentSnapshot,
currentSnapshot: currentSnapshot,
});
}
// cleanup and exit
Expand All @@ -80,92 +52,26 @@ export async function cmd({
}

async function pushSchema({
migrations,
migrationsDir,
statements,
appToken,
isDryRun,
currentSnapshot,
}: {
migrations: string[];
migrationsDir: URL;
statements: string[];
appToken: string;
isDryRun: boolean;
currentSnapshot: DBSnapshot;
}) {
// load all missing migrations
const initialSnapshot = migrations.find((m) => m === INITIAL_SNAPSHOT);
const filteredMigrations = migrations.filter((m) => m !== INITIAL_SNAPSHOT);
const missingMigrationContents = await Promise.all(
filteredMigrations.map((m) => loadMigration(m, migrationsDir))
);
// create a migration for the initial snapshot, if needed
const initialMigrationBatch = initialSnapshot
? (
await getMigrationQueries({
oldSnapshot: createEmptySnapshot(),
newSnapshot: await loadInitialSnapshot(migrationsDir),
})
).queries
: [];

// combine all missing migrations into a single batch
const confirmations = missingMigrationContents.reduce((acc, curr) => {
return [...acc, ...(curr.confirm || [])];
}, [] as string[]);
if (confirmations.length > 0) {
const response = await prompts([
...confirmations.map((message, index) => ({
type: 'confirm' as const,
name: String(index),
message: red('Warning: ') + message + '\nContinue?',
initial: true,
})),
]);
if (
Object.values(response).length === 0 ||
Object.values(response).some((value) => value === false)
) {
process.exit(1);
}
}

// combine all missing migrations into a single batch
const queries = missingMigrationContents.reduce((acc, curr) => {
return [...acc, ...curr.db];
}, initialMigrationBatch);
// apply the batch to the DB
await runMigrateQuery({ queries, migrations, snapshot: currentSnapshot, appToken, isDryRun });
}

async function runMigrateQuery({
queries: baseQueries,
migrations,
snapshot,
appToken,
isDryRun,
}: {
queries: string[];
migrations: string[];
snapshot: DBSnapshot;
appToken: string;
isDryRun?: boolean;
}) {
const queries = ['pragma defer_foreign_keys=true;', ...baseQueries];

const requestBody = {
snapshot,
migrations,
sql: queries,
snapshot: currentSnapshot,
sql: statements,
experimentalVersion: 1,
};

if (isDryRun) {
console.info('[DRY RUN] Batch query:', JSON.stringify(requestBody, null, 2));
return new Response(null, { status: 200 });
}

const url = new URL('/migrations/run', getRemoteDatabaseUrl());

const url = new URL('/db/push', getRemoteDatabaseUrl());
return await fetch(url, {
method: 'POST',
headers: new Headers({
Expand All @@ -174,28 +80,3 @@ async function runMigrateQuery({
body: JSON.stringify(requestBody),
});
}

async function prepareMigrateQuery({
migrations,
appToken,
}: {
migrations: string[];
appToken: string;
}) {
const url = new URL('/migrations/prepare', getRemoteDatabaseUrl());
const requestBody = {
migrations,
experimentalVersion: 1,
};
const result = await fetch(url, {
method: 'POST',
headers: new Headers({
Authorization: `Bearer ${appToken}`,
}),
body: JSON.stringify(requestBody),
});
if (result.status >= 400) {
throw new Error(await result.text());
}
return await result.json();
}
58 changes: 20 additions & 38 deletions packages/db/src/core/cli/commands/verify/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,52 +1,34 @@
import type { AstroConfig } from 'astro';
import type { Arguments } from 'yargs-parser';
import type { DBConfig } from '../../../types.js';
import { getMigrationQueries } from '../../migration-queries.js';
import {
MIGRATIONS_NOT_INITIALIZED,
MIGRATIONS_UP_TO_DATE,
MIGRATION_NEEDED,
getMigrationStatus,
} from '../../migrations.js';
import { getMigrationQueries,
createCurrentSnapshot,
createEmptySnapshot,
getProductionCurrentSnapshot, } from '../../migration-queries.js';
import { getManagedAppTokenOrExit } from '../../../tokens.js';

export async function cmd({
astroConfig,
dbConfig,
flags,
}: {
astroConfig: AstroConfig;
dbConfig: DBConfig;
flags: Arguments;
}) {
const status = await getMigrationStatus({ dbConfig, root: astroConfig.root });
const { state } = status;
if (flags.json) {
if (state === 'ahead') {
const { queries: migrationQueries } = await getMigrationQueries({
oldSnapshot: status.oldSnapshot,
newSnapshot: status.newSnapshot,
});
const newFileContent = {
diff: status.diff,
db: migrationQueries,
};
status.newFileContent = JSON.stringify(newFileContent, null, 2);
}
console.log(JSON.stringify(status));
process.exit(state === 'up-to-date' ? 0 : 1);
}
switch (state) {
case 'no-migrations-found': {
console.log(MIGRATIONS_NOT_INITIALIZED);
process.exit(1);
}
case 'ahead': {
console.log(MIGRATION_NEEDED);
process.exit(1);
}
case 'up-to-date': {
console.log(MIGRATIONS_UP_TO_DATE);
return;
}
const appToken = await getManagedAppTokenOrExit(flags.token);
const productionSnapshot = await getProductionCurrentSnapshot({ appToken: appToken.token });
const currentSnapshot = createCurrentSnapshot(dbConfig);
const { queries: migrationQueries } = await getMigrationQueries({
oldSnapshot: JSON.stringify(productionSnapshot) !== '{}' ? productionSnapshot : createEmptySnapshot(),
newSnapshot: currentSnapshot,
});

if (migrationQueries.length === 0) {
console.log(`Database schema is up to date.`);
} else {
console.log(`Database schema is out of date.`);
console.log(`Run 'astro db push' to push up your latest changes.`);
}

await appToken.destroy();
}
13 changes: 8 additions & 5 deletions packages/db/src/core/cli/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@ export async function cli({
const { cmd } = await import('./commands/shell/index.js');
return await cmd({ astroConfig, dbConfig, flags });
}
case 'gen':
case 'gen': {
console.log('"astro db gen" is no longer needed! Visit the docs for more information.');
return;
}
case 'sync': {
const { cmd } = await import('./commands/gen/index.js');
return await cmd({ astroConfig, dbConfig, flags });
console.log('"astro db sync" is no longer needed! Visit the docs for more information.');
return;
}
case 'push': {
const { cmd } = await import('./commands/push/index.js');
Expand Down Expand Up @@ -76,7 +79,7 @@ astro logout End your authenticated session with Astro Studio
astro link Link this directory to an Astro Studio project
astro db gen Creates snapshot based on your schema
astro db push Pushes migrations to Astro Studio
astro db verify Verifies migrations have been pushed and errors if not`;
astro db push Pushes schema updates to Astro Studio
astro db verify Tests schema updates /w Astro Studio (good for CI)`;
}
}
Loading

0 comments on commit 93ec9e2

Please sign in to comment.