Read migration files asynchrony

drizzle-orm/src/better-sqlite3/migrator.ts

@@ -1,11 +1,11 @@
 import type { MigrationConfig } from '~/migrator.ts';
-import { readMigrationFiles } from '~/migrator.ts';
+import { readMigrationFilesSync } from '~/migrator.ts';
 import type { BetterSQLite3Database } from './driver.ts';
 
 export function migrate<TSchema extends Record<string, unknown>>(
 	db: BetterSQLite3Database<TSchema>,
 	config: string | MigrationConfig,
 ) {
-	const migrations = readMigrationFiles(config);
+	const migrations = readMigrationFilesSync(config);
 	db.dialect.migrate(migrations, db.session, config);
 }

drizzle-orm/src/bun-sqlite/migrator.ts

@@ -1,11 +1,11 @@
 import type { MigrationConfig } from '~/migrator.ts';
-import { readMigrationFiles } from '~/migrator.ts';
+import { readMigrationFilesSync } from '~/migrator.ts';
 import type { BunSQLiteDatabase } from './driver.ts';
 
 export function migrate<TSchema extends Record<string, unknown>>(
 	db: BunSQLiteDatabase<TSchema>,
 	config: string | MigrationConfig,
 ) {
-	const migrations = readMigrationFiles(config);
+	const migrations = readMigrationFilesSync(config);
 	db.dialect.migrate(migrations, db.session, config);
 }

drizzle-orm/src/expo-sqlite/migrator.ts

@@ -9,10 +9,8 @@ interface MigrationConfig {
 	migrations: Record<string, string>;
 }
 
-async function readMigrationFiles({ journal, migrations }: MigrationConfig): Promise<MigrationMeta[]> {
-	const migrationQueries: MigrationMeta[] = [];
-
-	for await (const journalEntry of journal.entries) {
+function* readMigrationFiles({ journal, migrations }: MigrationConfig): IterableIterator<MigrationMeta> {
+	for (const journalEntry of journal.entries) {
 		const query = migrations[`m${journalEntry.idx.toString().padStart(4, '0')}`];
 
 		if (!query) {
@@ -24,18 +22,16 @@ async function readMigrationFiles({ journal, migrations }: MigrationConfig): Pro
 				return it;
 			});
 
-			migrationQueries.push({
+			yield {
 				sql: result,
 				bps: journalEntry.breakpoints,
 				folderMillis: journalEntry.when,
 				hash: '',
-			});
+			};
 		} catch {
 			throw new Error(`Failed to parse migration: ${journalEntry.tag}`);
 		}
 	}
-
-	return migrationQueries;
 }
 
 export async function migrate<TSchema extends Record<string, unknown>>(

drizzle-orm/src/libsql/migrator.ts

@@ -31,7 +31,7 @@ export async function migrate<TSchema extends Record<string, unknown>>(
 
 	const statementToBatch = [];
 
-	for (const migration of migrations) {
+	for await (const migration of migrations) {
 		if (!lastDbMigration || Number(lastDbMigration[2])! < migration.folderMillis) {
 			for (const stmt of migration.sql) {
 				statementToBatch.push(db.run(sql.raw(stmt)));

drizzle-orm/src/migrator.ts

@@ -1,5 +1,6 @@
 import crypto from 'node:crypto';
-import fs from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
+import fs from 'node:fs/promises';
 import path from 'node:path';
 
 export interface KitConfig {
@@ -20,10 +21,10 @@ export interface MigrationMeta {
 	bps: boolean;
 }
 
-export function readMigrationFiles(config: string | MigrationConfig): MigrationMeta[] {
+export async function* readMigrationFiles(config: string | MigrationConfig): AsyncIterableIterator<MigrationMeta> {
 	let migrationFolderTo: string | undefined;
 	if (typeof config === 'string') {
-		const configAsString = fs.readFileSync(path.resolve('.', config), 'utf8');
+		const configAsString = await fs.readFile(path.resolve('.', config), 'utf8');
 		const jsonConfig = JSON.parse(configAsString) as KitConfig;
 		migrationFolderTo = jsonConfig.out;
 	} else {
@@ -34,14 +35,59 @@ export function readMigrationFiles(config: string | MigrationConfig): MigrationM
 		throw new Error('no migration folder defined');
 	}
 
-	const migrationQueries: MigrationMeta[] = [];
+	let journalAsString: string
+	try {
+		journalAsString = await fs.readFile(`${migrationFolderTo}/meta/_journal.json`).toString();
+	} catch {
+		throw new Error(`Can't find meta/_journal.json file`);
+	}
+
+	const journal = JSON.parse(journalAsString) as {
+		entries: { idx: number; when: number; tag: string; breakpoints: boolean }[];
+	};
+
+	for (const journalEntry of journal.entries) {
+		const migrationPath = `${migrationFolderTo}/${journalEntry.tag}.sql`;
+
+		try {
+			const query = await fs.readFile(`${migrationFolderTo}/${journalEntry.tag}.sql`).toString();
+
+			const result = query.split('--> statement-breakpoint').map((it) => {
+				return it;
+			});
+
+			yield {
+				sql: result,
+				bps: journalEntry.breakpoints,
+				folderMillis: journalEntry.when,
+				hash: crypto.createHash('sha256').update(query).digest('hex'),
+			};
+		} catch {
+			throw new Error(`No file ${migrationPath} found in ${migrationFolderTo} folder`);
+		}
+	}
+}
+
+export function* readMigrationFilesSync(config: string | MigrationConfig): IterableIterator<MigrationMeta> {
+	let migrationFolderTo: string | undefined;
+	if (typeof config === 'string') {
+		const configAsString = readFileSync(path.resolve('.', config), 'utf8');
+		const jsonConfig = JSON.parse(configAsString) as KitConfig;
+		migrationFolderTo = jsonConfig.out;
+	} else {
+		migrationFolderTo = config.migrationsFolder;
+	}
+
+	if (!migrationFolderTo) {
+		throw new Error('no migration folder defined');
+	}
 
 	const journalPath = `${migrationFolderTo}/meta/_journal.json`;
-	if (!fs.existsSync(journalPath)) {
+	if (!existsSync(journalPath)) {
 		throw new Error(`Can't find meta/_journal.json file`);
 	}
 
-	const journalAsString = fs.readFileSync(`${migrationFolderTo}/meta/_journal.json`).toString();
+	const journalAsString = readFileSync(`${migrationFolderTo}/meta/_journal.json`).toString();
 
 	const journal = JSON.parse(journalAsString) as {
 		entries: { idx: number; when: number; tag: string; breakpoints: boolean }[];
@@ -51,22 +97,20 @@ export function readMigrationFiles(config: string | MigrationConfig): MigrationM
 		const migrationPath = `${migrationFolderTo}/${journalEntry.tag}.sql`;
 
 		try {
-			const query = fs.readFileSync(`${migrationFolderTo}/${journalEntry.tag}.sql`).toString();
+			const query = readFileSync(`${migrationFolderTo}/${journalEntry.tag}.sql`).toString();
 
 			const result = query.split('--> statement-breakpoint').map((it) => {
 				return it;
 			});
 
-			migrationQueries.push({
+			yield {
 				sql: result,
 				bps: journalEntry.breakpoints,
 				folderMillis: journalEntry.when,
 				hash: crypto.createHash('sha256').update(query).digest('hex'),
-			});
+			};
 		} catch {
 			throw new Error(`No file ${migrationPath} found in ${migrationFolderTo} folder`);
 		}
 	}
-
-	return migrationQueries;
 }

drizzle-orm/src/mysql-core/dialect.ts

@@ -32,7 +32,7 @@ export class MySqlDialect {
 	static readonly [entityKind]: string = 'MySqlDialect';
 
 	async migrate(
-		migrations: MigrationMeta[],
+		migrations: AsyncIterableIterator<MigrationMeta>,
 		session: MySqlSession,
 		config: Omit<MigrationConfig, 'migrationsSchema'>,
 	): Promise<void> {
@@ -53,7 +53,7 @@ export class MySqlDialect {
 		const lastDbMigration = dbMigrations[0];
 
 		await session.transaction(async (tx) => {
-			for (const migration of migrations) {
+			for await (const migration of migrations) {
 				if (
 					!lastDbMigration
 					|| Number(lastDbMigration.created_at) < migration.folderMillis

drizzle-orm/src/mysql-proxy/migrator.ts

@@ -34,7 +34,7 @@ export async function migrate<TSchema extends Record<string, unknown>>(
 
 	const queriesToRun: string[] = [];
 
-	for (const migration of migrations) {
+	for await (const migration of migrations) {
 		if (
 			!lastDbMigration
 			|| Number(lastDbMigration.created_at) < migration.folderMillis

drizzle-orm/src/op-sqlite/migrator.ts

@@ -9,10 +9,8 @@ interface MigrationConfig {
 	migrations: Record<string, string>;
 }
 
-async function readMigrationFiles({ journal, migrations }: MigrationConfig): Promise<MigrationMeta[]> {
-	const migrationQueries: MigrationMeta[] = [];
-
-	for await (const journalEntry of journal.entries) {
+async function* readMigrationFiles({ journal, migrations }: MigrationConfig): AsyncIterableIterator<MigrationMeta> {
+	for (const journalEntry of journal.entries) {
 		const query = migrations[`m${journalEntry.idx.toString().padStart(4, '0')}`];
 
 		if (!query) {
@@ -24,18 +22,16 @@ async function readMigrationFiles({ journal, migrations }: MigrationConfig): Pro
 				return it;
 			});
 
-			migrationQueries.push({
+			yield {
 				sql: result,
 				bps: journalEntry.breakpoints,
 				folderMillis: journalEntry.when,
 				hash: '',
-			});
+			};
 		} catch {
 			throw new Error(`Failed to parse migration: ${journalEntry.tag}`);
 		}
 	}
-
-	return migrationQueries;
 }
 
 export async function migrate<TSchema extends Record<string, unknown>>(

drizzle-orm/src/pg-core/dialect.ts

@@ -46,7 +46,7 @@ import type { PgMaterializedView } from './view.ts';
 export class PgDialect {
 	static readonly [entityKind]: string = 'PgDialect';
 
-	async migrate(migrations: MigrationMeta[], session: PgSession, config: string | MigrationConfig): Promise<void> {
+	async migrate(migrations: AsyncIterableIterator<MigrationMeta>, session: PgSession, config: string | MigrationConfig): Promise<void> {
 		const migrationsTable = typeof config === 'string'
 			? '__drizzle_migrations'
 			: config.migrationsTable ?? '__drizzle_migrations';

drizzle-orm/src/pg-proxy/migrator.ts

@@ -35,7 +35,7 @@ export async function migrate<TSchema extends Record<string, unknown>>(
 
 	const queriesToRun: string[] = [];
 
-	for (const migration of migrations) {
+	for await (const migration of migrations) {
 		if (
 			!lastDbMigration
 			|| Number(lastDbMigration.created_at)! < migration.folderMillis

drizzle-orm/src/sql-js/migrator.ts

@@ -1,11 +1,11 @@
 import type { MigrationConfig } from '~/migrator.ts';
-import { readMigrationFiles } from '~/migrator.ts';
+import { readMigrationFilesSync } from '~/migrator.ts';
 import type { SQLJsDatabase } from './driver.ts';
 
 export function migrate<TSchema extends Record<string, unknown>>(
 	db: SQLJsDatabase<TSchema>,
 	config: string | MigrationConfig,
 ) {
-	const migrations = readMigrationFiles(config);
+	const migrations = readMigrationFilesSync(config);
 	db.dialect.migrate(migrations, db.session, config);
 }

drizzle-orm/src/sqlite-core/dialect.ts

@@ -717,7 +717,7 @@ export class SQLiteSyncDialect extends SQLiteDialect {
 	static readonly [entityKind]: string = 'SQLiteSyncDialect';
 
 	migrate(
-		migrations: MigrationMeta[],
+		migrations: IterableIterator<MigrationMeta>,
 		session: SQLiteSession<'sync', unknown, Record<string, unknown>, TablesRelationalConfig>,
 		config?: string | MigrationConfig,
 	): void {
@@ -769,7 +769,7 @@ export class SQLiteAsyncDialect extends SQLiteDialect {
 	static readonly [entityKind]: string = 'SQLiteAsyncDialect';
 
 	async migrate(
-		migrations: MigrationMeta[],
+		migrations: AsyncIterableIterator<MigrationMeta>,
 		session: SQLiteSession<'async', unknown, Record<string, unknown>, TablesRelationalConfig>,
 		config?: string | MigrationConfig,
 	): Promise<void> {
@@ -795,7 +795,7 @@ export class SQLiteAsyncDialect extends SQLiteDialect {
 		const lastDbMigration = dbMigrations[0] ?? undefined;
 
 		await session.transaction(async (tx) => {
-			for (const migration of migrations) {
+			for await (const migration of migrations) {
 				if (!lastDbMigration || Number(lastDbMigration[2])! < migration.folderMillis) {
 					for (const stmt of migration.sql) {
 						await tx.run(sql.raw(stmt));

drizzle-orm/src/sqlite-proxy/migrator.ts

@@ -33,7 +33,7 @@ export async function migrate<TSchema extends Record<string, unknown>>(
 	const lastDbMigration = dbMigrations[0] ?? undefined;
 
 	const queriesToRun: string[] = [];
-	for (const migration of migrations) {
+	for await (const migration of migrations) {
 		if (
 			!lastDbMigration
 			|| Number(lastDbMigration[2])! < migration.folderMillis