Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[project-s] Phraseをストアで持つようにする #1663

Merged
merged 2 commits into from
Dec 3, 2023
Merged

[project-s] Phraseをストアで持つようにする #1663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7b2aa04
phraseをvuexのstateで持つように
sigprogramming Dec 1, 2023
79a8683
exportingAudioをnowAudioExportingに変更
sigprogramming Dec 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 127 additions & 92 deletions src/store/singing.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
SingingStoreTypes,
SaveResultObject,
Singer,
Phrase,
} from "./type";
import { createPartialStore } from "./vuex";
import { createUILockAction } from "./ui";
Expand Down Expand Up @@ -179,29 +180,13 @@ const createPromiseThatResolvesWhen = (
});
};

type Phrase = {
readonly singer: Singer | undefined;
readonly score: Score;
// renderingが進むに連れてデータが代入されていく
query?: AudioQuery;
queryHash?: string; // queryの変更を検知するためのハッシュ
blob?: Blob;
startTime?: number;
source?: Instrument | AudioPlayer; // ひとまずPhraseに持たせる
sequence?: Sequence; // ひとまずPhraseに持たせる
};

const generateSingerAndScoreHash = async (obj: {
singer: Singer | undefined;
score: Score;
}) => {
return _generateHash(obj);
};

const generateAudioQueryHash = async (obj: AudioQuery) => {
return _generateHash(obj);
};

const isValidTpqn = (tpqn: number) => {
return (
Number.isInteger(tpqn) &&
Expand Down Expand Up @@ -293,8 +278,14 @@ if (window.AudioContext) {
clipper.output.connect(audioContext.destination);
}

type PhraseData = {
blob?: Blob;
source?: Instrument | AudioPlayer; // ひとまずPhraseDataに持たせる
sequence?: Sequence; // ひとまずPhraseDataに持たせる
};
Comment on lines +281 to +285
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

おそらくこのデータは、Vuex内で閉じていて他で使ってほしくない値というニュアンスなのかなと思いました！
ローカル変数の概念が無いのでそう分けられたのかなと。

となると、コメントでこの型の説明付けておくとニュアンス伝わりやすいかもです。こんな感じとか？

Suggested change
type PhraseData = {
blob?: Blob;
source?: Instrument | AudioPlayer; // ひとまずPhraseDataに持たせる
sequence?: Sequence; // ひとまずPhraseDataに持たせる
};
// Phraseのstore内用データ
type PhraseData = {
blob?: Blob;
source?: Instrument | AudioPlayer; // ひとまずPhraseDataに持たせる
sequence?: Sequence; // ひとまずPhraseDataに持たせる
};


const playheadPosition = new FrequentlyUpdatedState(0);
const allPhrases = new Map<string, Phrase>();
const phraseDataMap = new Map<string, PhraseData>();
const phraseAudioBlobCache = new Map<string, Blob>();
Comment on lines -297 to 289
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PhraseDataにもblobがあって、それとは別にphraseAudioBlobCacheもあるのでちょっとややこしいかもと思いました！

const animationFrameRunner = new AnimationFrameRunner();

Expand All @@ -317,6 +308,7 @@ export const singingStoreState: SingingStoreState = {
],
notes: [],
},
phrases: {},
// NOTE: UIの状態は試行のためsinging.tsに局所化する+Hydrateが必要
isShowSinger: true,
sequencerZoomX: 0.5,
Expand All @@ -332,7 +324,7 @@ export const singingStoreState: SingingStoreState = {
startRenderingRequested: false,
stopRenderingRequested: false,
nowRendering: false,
exportingAudio: false,
nowAudioExporting: false,
cancellationOfAudioExportRequested: false,
};

Expand Down Expand Up @@ -679,6 +671,39 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
},
},

SET_PHRASE: {
mutation(
state,
{ phraseKey, phrase }: { phraseKey: string; phrase: Phrase }
) {
state.phrases[phraseKey] = phrase;
},
},

DELETE_PHRASE: {
mutation(state, { phraseKey }: { phraseKey: string }) {
delete state.phrases[phraseKey];
},
},

SET_AUDIO_QUERY_TO_PHRASE: {
mutation(
state,
{ phraseKey, audioQuery }: { phraseKey: string; audioQuery: AudioQuery }
) {
state.phrases[phraseKey].query = audioQuery;
},
},

SET_START_TIME_TO_PHRASE: {
mutation(
state,
{ phraseKey, startTime }: { phraseKey: string; startTime: number }
) {
state.phrases[phraseKey].startTime = startTime;
},
},

SET_SNAP_TYPE: {
mutation(state, { snapType }) {
state.sequencerSnapType = snapType;
Expand Down Expand Up @@ -1077,7 +1102,7 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
return query;
};

const calculateStartTime = (score: Score, query: AudioQuery) => {
const calcStartTime = (score: Score, query: AudioQuery) => {
const firstMora = query.accentPhrases[0].moras[0];
let startTime = tickToSecond(
score.notes[0].position,
Expand Down Expand Up @@ -1133,8 +1158,10 @@ export const singingStore = createPartialStore<SingingStoreTypes>({

const foundPhrases = await searchPhrases(singer, score);
for (const [hash, phrase] of foundPhrases) {
if (!allPhrases.has(hash)) {
allPhrases.set(hash, phrase);
const key = hash;
if (!state.phrases[key]) {
commit("SET_PHRASE", { phraseKey: key, phrase });

// フレーズ追加時の処理
const noteEvents = generateNoteEvents(
phrase.score.notes,
Expand All @@ -1149,21 +1176,28 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
noteEvents,
};
transportRef.addSequence(noteSequence);

phrase.source = polySynth;
phrase.sequence = noteSequence;
phraseDataMap.set(key, {
source: polySynth,
sequence: noteSequence,
});
}
}
for (const [hash, phrase] of allPhrases) {
if (!foundPhrases.has(hash)) {
allPhrases.delete(hash);
for (const key of Object.keys(state.phrases)) {
if (!foundPhrases.has(key)) {
commit("DELETE_PHRASE", { phraseKey: key });

// フレーズ削除時の処理
if (phrase.source) {
phrase.source.output.disconnect();
const phraseData = phraseDataMap.get(key);
if (!phraseData) {
throw new Error("phraseData is undefined");
}
if (phraseData.source) {
phraseData.source.output.disconnect();
}
if (phrase.sequence) {
transportRef.removeSequence(phrase.sequence);
if (phraseData.sequence) {
transportRef.removeSequence(phraseData.sequence);
}
phraseDataMap.delete(key);
}
}

Expand All @@ -1173,20 +1207,25 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
return;
}

for (const phrase of allPhrases.values()) {
for (const [key, phrase] of Object.entries(state.phrases)) {
if (!phrase.singer) {
continue;
}

// Phrase -> AudioQuery
// Singer & Score -> AudioQuery
// Score & AudioQuery -> StartTime

if (!phrase.query) {
window.electron.logInfo(`Generating query...`);

phrase.query = await generateAndEditQuery(
const audioQuery = await generateAndEditQuery(
phrase.singer,
phrase.score
);
commit("SET_AUDIO_QUERY_TO_PHRASE", { phraseKey: key, audioQuery });

const startTime = calcStartTime(phrase.score, audioQuery);
commit("SET_START_TIME_TO_PHRASE", { phraseKey: key, startTime });
Comment on lines +1225 to +1228
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

audioQueryとstartTimeは片方が存在すればもう片方は必ず存在する気がするので、それを型に入れてあげるとundefinedチェック用のifを二回実行しなくて済むかもと思いました！
つまり、こうでも良いかも、と。

type Phrase = {
  otherData?: {
    audioQuery: AudioQuery,
    startTime: number
  }
}

まあこのあたり今はどういう形が最高かわからないのですが、こういう考え方もあるよという感じで･･･！


window.electron.logInfo(`Query generated.`);
}
Expand All @@ -1196,38 +1235,43 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
}

// AudioQuery -> Blob
// Phrase & AudioQuery -> startTime
// Blob & StartTime -> AudioSequence

const queryHash = await generateAudioQueryHash(phrase.query);
// クエリが変更されていたら再合成
if (queryHash !== phrase.queryHash) {
phrase.blob = phraseAudioBlobCache.get(queryHash);
if (phrase.blob) {
const phraseData = phraseDataMap.get(key);
if (!phraseData) {
throw new Error("phraseData is undefined");
}
if (!phrase.query) {
throw new Error("query is undefined.");
}
if (phrase.startTime === undefined) {
throw new Error("startTime is undefined.");
}
if (!phraseData.blob) {
phraseData.blob = phraseAudioBlobCache.get(key);
if (phraseData.blob) {
window.electron.logInfo(`Loaded audio buffer from cache.`);
} else {
window.electron.logInfo(`Synthesizing...`);

phrase.blob = await synthesize(phrase.singer, phrase.query);
phraseAudioBlobCache.set(queryHash, phrase.blob);
phraseData.blob = await synthesize(phrase.singer, phrase.query);
phraseAudioBlobCache.set(key, phraseData.blob);

window.electron.logInfo(`Synthesized.`);
}
phrase.queryHash = queryHash;
phrase.startTime = calculateStartTime(phrase.score, phrase.query);

// 音源とシーケンスを作成し直して、再接続する
if (phrase.source) {
phrase.source.output.disconnect();
if (phraseData.source) {
phraseData.source.output.disconnect();
}
if (phrase.sequence) {
transportRef.removeSequence(phrase.sequence);
if (phraseData.sequence) {
transportRef.removeSequence(phraseData.sequence);
}

const audioPlayer = new AudioPlayer(audioContextRef);
const audioEvents = await generateAudioEvents(
audioContextRef,
phrase.startTime,
phrase.blob
phraseData.blob
);
const audioSequence: AudioSequence = {
type: "audio",
Expand All @@ -1236,9 +1280,8 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
};
audioPlayer.output.connect(channelStripRef.input);
transportRef.addSequence(audioSequence);

phrase.source = audioPlayer;
phrase.sequence = audioSequence;
phraseData.source = audioPlayer;
phraseData.sequence = audioSequence;
}

if (startRenderingRequested() || stopRenderingRequested()) {
Expand Down Expand Up @@ -1780,9 +1823,9 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
),
},

SET_EXPORTING_AUDIO: {
mutation(state, { exportingAudio }) {
state.exportingAudio = exportingAudio;
SET_NOW_AUDIO_EXPORTING: {
mutation(state, { nowAudioExporting }) {
state.nowAudioExporting = nowAudioExporting;
},
},

Expand Down Expand Up @@ -1940,39 +1983,31 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
: undefined;
const clipper = new Clipper(offlineAudioContext);

for (const phrase of allPhrases.values()) {
// TODO: この辺りの処理を共通化する
if (phrase.startTime !== undefined && phrase.blob) {
// レンダリング済みのフレーズの場合
const audioEvents = await generateAudioEvents(
offlineAudioContext,
phrase.startTime,
phrase.blob
);
const audioPlayer = new AudioPlayer(offlineAudioContext);
audioPlayer.output.connect(channelStrip.input);
const audioSequence: AudioSequence = {
type: "audio",
audioPlayer,
audioEvents,
};
offlineTransport.addSequence(audioSequence);
} else {
// レンダリング未完了のフレーズの場合
const noteEvents = generateNoteEvents(
phrase.score.notes,
phrase.score.tempos,
phrase.score.tpqn
);
const polySynth = new PolySynth(offlineAudioContext);
polySynth.output.connect(channelStrip.input);
const noteSequence: NoteSequence = {
type: "note",
instrument: polySynth,
noteEvents,
};
offlineTransport.addSequence(noteSequence);
for (const [key, phrase] of Object.entries(state.phrases)) {
const phraseData = phraseDataMap.get(key);
if (!phraseData) {
throw new Error("phraseData is undefined");
}
if (phrase.startTime === undefined) {
throw new Error("startTime is undefined");
}
if (!phraseData.blob) {
throw new Error("blob is undefined");
}
// TODO: この辺りの処理を共通化する
const audioEvents = await generateAudioEvents(
offlineAudioContext,
phrase.startTime,
phraseData.blob
);
const audioPlayer = new AudioPlayer(offlineAudioContext);
audioPlayer.output.connect(channelStrip.input);
const audioSequence: AudioSequence = {
type: "audio",
audioPlayer,
audioEvents,
};
offlineTransport.addSequence(audioSequence);
}
channelStrip.volume = 1;
if (limiter) {
Expand Down Expand Up @@ -2017,21 +2052,21 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
return { result: "SUCCESS", path: filePath };
};

commit("SET_EXPORTING_AUDIO", { exportingAudio: true });
commit("SET_NOW_AUDIO_EXPORTING", { nowAudioExporting: true });
return exportWaveFile().finally(() => {
commit("SET_CANCELLATION_OF_AUDIO_EXPORT_REQUESTED", {
cancellationOfAudioExportRequested: false,
});
commit("SET_EXPORTING_AUDIO", { exportingAudio: false });
commit("SET_NOW_AUDIO_EXPORTING", { nowAudioExporting: false });
});
}
),
},

CANCEL_AUDIO_EXPORT: {
async action({ state, commit, dispatch }) {
if (!state.exportingAudio) {
dispatch("LOG_WARN", "CANCEL_AUDIO_EXPORT on !exportingAudio");
if (!state.nowAudioExporting) {
dispatch("LOG_WARN", "CANCEL_AUDIO_EXPORT on !nowAudioExporting");
return;
}
commit("SET_CANCELLATION_OF_AUDIO_EXPORT_REQUESTED", {
Expand Down
Loading
Loading