feat(backend): リアルタイム更新時のノート取得にCDNキャッシュを併用できるように

CHANGELOG.md

@@ -15,6 +15,9 @@
 - Fix: 絵文字管理画面で一部の絵文字が表示されない問題を修正
 
 ### Server
+- Feat: サーバーがCDNを利用している場合、タイムラインのリアルタイム更新時にCDNのキャッシュを併用できるように（上級者向け）
+  - この機能はCDNの設定と連携するように設計されています。Misskey側のみで機能を有効にし、CDNを正しく設定していない場合、かえって負荷が大きくなる可能性があります。詳しくはMisskey Hubのドキュメント (TODO) を参照してください。
+  - Based on https://github.com/MisskeyIO/misskey/pull/834, https://github.com/MisskeyIO/misskey/pull/851, https://github.com/MisskeyIO/misskey/pull/853
 - Fix: ユーザーのプロフィール画面をアドレス入力などで直接表示した際に概要タブの描画に失敗する問題の修正( #15032 )
 - Fix: 起動前の疎通チェックが機能しなくなっていた問題を修正  
   (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/737)

locales/index.d.ts

@@ -5222,6 +5222,14 @@ export interface Locale extends ILocale {
      * 注意事項を理解した上でオンにします。
      */
     "acknowledgeNotesAndEnable": string;
+    /**
+     * リアルタイム更新時のノート取得にCDNキャッシュを併用する
+     */
+    "enableStreamNotesCdnCache": string;
+    /**
+     * （上級者向け）サーバーでCDNを使用している場合は、この設定をオンにしたうえでCDNの設定を適切に調整するとサーバーへの負荷を軽減できます。WebSocket通信ではリアルタイム性のある情報のみを送信し、残りの情報はCDNキャッシュから取得して突合します。このため、クライアント側の負荷やデータ通信量に影響が出る可能性があります。
+     */
+    "enableStreamNotesCdnCacheDescription": string;
     "_accountSettings": {
         /**
          * コンテンツの表示にログインを必須にする

locales/ja-JP.yml

@@ -1301,6 +1301,8 @@ lockdown: "ロックダウン"
 pleaseSelectAccount: "アカウントを選択してください"
 availableRoles: "利用可能なロール"
 acknowledgeNotesAndEnable: "注意事項を理解した上でオンにします。"
+enableStreamNotesCdnCache: "リアルタイム更新時のノート取得にCDNキャッシュを併用する"
+enableStreamNotesCdnCacheDescription: "（上級者向け）サーバーでCDNを使用している場合は、この設定をオンにしたうえでCDNの設定を適切に調整するとサーバーへの負荷を軽減できます。WebSocket通信ではリアルタイム性のある情報のみを送信し、残りの情報はCDNキャッシュから取得して突合します。このため、クライアント側の負荷やデータ通信量に影響が出る可能性があります。"
 
 _accountSettings:
   requireSigninToViewContents: "コンテンツの表示にログインを必須にする"

packages/backend/migration/1735377503979-StreamNotesCdnCache.js

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class StreamNotesCdnCache1735377503979 {
+    name = 'StreamNotesCdnCache1735377503979'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "enableStreamNotesCdnCache" boolean NOT NULL DEFAULT false`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableStreamNotesCdnCache"`);
+    }
+}

packages/backend/src/core/entities/MetaEntityService.ts

@@ -130,6 +130,7 @@ export class MetaEntityService {
 
 			mediaProxy: this.config.mediaProxy,
 			enableUrlPreview: instance.urlPreviewEnabled,
+			enableStreamNotesCdnCache: instance.enableStreamNotesCdnCache,
 			noteSearchableScope: (this.config.meilisearch == null || this.config.meilisearch.scope !== 'local') ? 'global' : 'local',
 			maxFileSize: this.config.maxFileSize,
 		};

packages/backend/src/core/entities/NoteEntityService.ts

@@ -325,6 +325,17 @@ export class NoteEntityService implements OnModuleInit {
 		return true;
 	}
 
+	/** CDNなどにキャッシュしても問題ないノートかどうか */
+	@bindThis
+	public canCache(note: MiNote | Packed<'Note'>): boolean {
+		return (
+			(note.visibility === 'public' || note.visibility === 'home') &&
+			note.user?.makeNotesFollowersOnlyBefore == null &&
+			note.user?.makeNotesHiddenBefore == null &&
+			(note.user?.requireSigninToViewContents === false || note.user?.requireSigninToViewContents == null)
+		);
+	}
+
 	@bindThis
 	public async packAttachedFiles(fileIds: MiNote['fileIds'], packedFiles: Map<MiNote['fileIds'][number], Packed<'DriveFile'> | null>): Promise<Packed<'DriveFile'>[]> {
 		const missingIds = [];

packages/backend/src/models/Meta.ts

@@ -609,6 +609,11 @@ export class MiMeta {
 	})
 	public enableReactionsBuffering: boolean;
 
+	@Column('boolean', {
+		default: false,
+	})
+	public enableStreamNotesCdnCache: boolean;
+
 	@Column('integer', {
 		default: 0,
 	})

packages/backend/src/models/json-schema/meta.ts

@@ -215,6 +215,10 @@ export const packedMetaLiteSchema = {
 			type: 'boolean',
 			optional: false, nullable: false,
 		},
+		enableStreamNotesCdnCache: {
+			type: 'boolean',
+			optional: false, nullable: false,
+		},
 		backgroundImageUrl: {
 			type: 'string',
 			optional: false, nullable: true,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -396,6 +396,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			enableStreamNotesCdnCache: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
 			notesPerOneAd: {
 				type: 'number',
 				optional: false, nullable: false,
@@ -652,6 +656,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax,
 				perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax,
 				enableReactionsBuffering: instance.enableReactionsBuffering,
+				enableStreamNotesCdnCache: instance.enableStreamNotesCdnCache,
 				notesPerOneAd: instance.notesPerOneAd,
 				summalyProxy: instance.urlPreviewSummaryProxyUrl,
 				urlPreviewEnabled: instance.urlPreviewEnabled,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -150,6 +150,7 @@ export const paramDef = {
 		perUserHomeTimelineCacheMax: { type: 'integer' },
 		perUserListTimelineCacheMax: { type: 'integer' },
 		enableReactionsBuffering: { type: 'boolean' },
+		enableStreamNotesCdnCache: { type: 'boolean' },
 		notesPerOneAd: { type: 'integer' },
 		silencedHosts: {
 			type: 'array',
@@ -631,6 +632,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableReactionsBuffering = ps.enableReactionsBuffering;
 			}
 
+			if (ps.enableStreamNotesCdnCache !== undefined) {
+				set.enableStreamNotesCdnCache = ps.enableStreamNotesCdnCache;
+			}
+
 			if (ps.notesPerOneAd !== undefined) {
 				set.notesPerOneAd = ps.notesPerOneAd;
 			}

packages/backend/src/server/api/stream/channels/antenna.ts

@@ -16,6 +16,7 @@ class AntennaChannel extends Channel {
 	public static requireCredential = true as const;
 	public static kind = 'read:account';
 	private antennaId: string;
+	private minimize: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -30,7 +31,8 @@ class AntennaChannel extends Channel {
 	@bindThis
 	public async init(params: JsonObject) {
 		if (typeof params.antennaId !== 'string') return;
-		this.antennaId = params.antennaId;
+		this.antennaId = params.antennaId as string;
+		this.minimize = !!(params.minimize ?? false);
 
 		// Subscribe stream
 		this.subscriber.on(`antennaStream:${this.antennaId}`, this.onEvent);
@@ -43,9 +45,25 @@ class AntennaChannel extends Channel {
 
 			if (this.isNoteMutedOrBlocked(note)) return;
 
-			this.connection.cacheNote(note);
-
-			this.send('note', note);
+			if (this.minimize) {
+				if (this.noteEntityService.canCache(note)) {
+					this.send('note', {
+						id: note.id, myReaction: note.myReaction,
+						poll: note.poll?.choices ? { choices: note.poll.choices } : undefined,
+						reply: note.reply?.myReaction ? { myReaction: note.reply.myReaction } : undefined,
+						renote: note.renote?.myReaction ? { myReaction: note.renote.myReaction } : undefined,
+						_allowCached_: true,
+					});
+				} else {
+					this.send('note', {
+						id: note.id,
+						_allowCached_: false,
+					});
+				}
+			} else {
+				this.connection.cacheNote(note);
+				this.send('note', note);
+			}
 		} else {
 			this.send(data.type, data.body);
 		}

packages/backend/src/server/api/stream/channels/channel.ts

@@ -16,6 +16,7 @@ class ChannelChannel extends Channel {
 	public static shouldShare = false;
 	public static requireCredential = false as const;
 	private channelId: string;
+	private minimize: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -30,7 +31,8 @@ class ChannelChannel extends Channel {
 	@bindThis
 	public async init(params: JsonObject) {
 		if (typeof params.channelId !== 'string') return;
-		this.channelId = params.channelId;
+		this.channelId = params.channelId as string;
+		this.minimize = !!(params.minimize ?? false);
 
 		// Subscribe stream
 		this.subscriber.on('notesStream', this.onNote);
@@ -49,9 +51,25 @@ class ChannelChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
-
-		this.send('note', note);
+		if (this.minimize) {
+			if (this.noteEntityService.canCache(note)) {
+				this.send('note', {
+					id: note.id, myReaction: note.myReaction,
+					poll: note.poll?.choices ? { choices: note.poll.choices } : undefined,
+					reply: note.reply?.myReaction ? { myReaction: note.reply.myReaction } : undefined,
+					renote: note.renote?.myReaction ? { myReaction: note.renote.myReaction } : undefined,
+					_allowCached_: true,
+				});
+			} else {
+				this.send('note', {
+					id: note.id,
+					_allowCached_: false,
+				});
+			}
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/global-timeline.ts

@@ -19,6 +19,7 @@ class GlobalTimelineChannel extends Channel {
 	public static requireCredential = false as const;
 	private withRenotes: boolean;
 	private withFiles: boolean;
+	private minimize: boolean;
 
 	constructor(
 		private metaService: MetaService,
@@ -39,6 +40,7 @@ class GlobalTimelineChannel extends Channel {
 
 		this.withRenotes = !!(params.withRenotes ?? true);
 		this.withFiles = !!(params.withFiles ?? false);
+		this.minimize = !!(params.minimize ?? false);
 
 		// Subscribe events
 		this.subscriber.on('notesStream', this.onNote);
@@ -62,9 +64,25 @@ class GlobalTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
-
-		this.send('note', note);
+		if (this.minimize) {
+			if (this.noteEntityService.canCache(note)) {
+				this.send('note', {
+					id: note.id, myReaction: note.myReaction,
+					poll: note.poll?.choices ? { choices: note.poll.choices } : undefined,
+					reply: note.reply?.myReaction ? { myReaction: note.reply.myReaction } : undefined,
+					renote: note.renote?.myReaction ? { myReaction: note.renote.myReaction } : undefined,
+					_allowCached_: true,
+				});
+			} else {
+				this.send('note', {
+					id: note.id,
+					_allowCached_: false,
+				});
+			}
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/home-timeline.ts

@@ -18,6 +18,7 @@ class HomeTimelineChannel extends Channel {
 	public static kind = 'read:account';
 	private withRenotes: boolean;
 	private withFiles: boolean;
+	private minimize: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -33,6 +34,7 @@ class HomeTimelineChannel extends Channel {
 	public async init(params: JsonObject) {
 		this.withRenotes = !!(params.withRenotes ?? true);
 		this.withFiles = !!(params.withFiles ?? false);
+		this.minimize = !!(params.minimize ?? false);
 
 		this.subscriber.on('notesStream', this.onNote);
 	}
@@ -86,9 +88,25 @@ class HomeTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
-
-		this.send('note', note);
+		if (this.minimize) {
+			if (this.noteEntityService.canCache(note)) {
+				this.send('note', {
+					id: note.id, myReaction: note.myReaction,
+					poll: note.poll?.choices ? { choices: note.poll.choices } : undefined,
+					reply: note.reply?.myReaction ? { myReaction: note.reply.myReaction } : undefined,
+					renote: note.renote?.myReaction ? { myReaction: note.renote.myReaction } : undefined,
+					_allowCached_: true,
+				});
+			} else {
+				this.send('note', {
+					id: note.id,
+					_allowCached_: false,
+				});
+			}
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/hybrid-timeline.ts

@@ -21,6 +21,7 @@ class HybridTimelineChannel extends Channel {
 	private withRenotes: boolean;
 	private withReplies: boolean;
 	private withFiles: boolean;
+	private minimize: boolean;
 
 	constructor(
 		private metaService: MetaService,
@@ -42,6 +43,7 @@ class HybridTimelineChannel extends Channel {
 		this.withRenotes = !!(params.withRenotes ?? true);
 		this.withReplies = !!(params.withReplies ?? false);
 		this.withFiles = !!(params.withFiles ?? false);
+		this.minimize = !!(params.minimize ?? false);
 
 		// Subscribe events
 		this.subscriber.on('notesStream', this.onNote);
@@ -101,9 +103,25 @@ class HybridTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
-
-		this.send('note', note);
+		if (this.minimize) {
+			if (this.noteEntityService.canCache(note)) {
+				this.send('note', {
+					id: note.id, myReaction: note.myReaction,
+					poll: note.poll?.choices ? { choices: note.poll.choices } : undefined,
+					reply: note.reply?.myReaction ? { myReaction: note.reply.myReaction } : undefined,
+					renote: note.renote?.myReaction ? { myReaction: note.renote.myReaction } : undefined,
+					_allowCached_: true,
+				});
+			} else {
+				this.send('note', {
+					id: note.id,
+					_allowCached_: false,
+				});
+			}
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis