From 6e274063527e840f5485b52dac5536acef6ee0db Mon Sep 17 00:00:00 2001 From: Minsoo Cheong Date: Mon, 25 Mar 2024 20:22:02 +0900 Subject: [PATCH 1/4] embedding: assign `n_ubatch` value, print error on `n_batch` overflow --- examples/embedding/embedding.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index cbf9aa2b560dd..f2dc38a817118 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -61,6 +61,8 @@ int main(int argc, char ** argv) { } params.embedding = true; + // For BERT models, batch size must be equal to ubatch size + params.n_ubatch = params.n_batch; print_build_info(); @@ -114,7 +116,9 @@ int main(int argc, char ** argv) { for (const auto & prompt : prompts) { auto inp = ::llama_tokenize(ctx, prompt, true, false); if (inp.size() > n_batch) { - inp.resize(n_batch); + fprintf(stderr, "%s: error: number of tokens in input line (%lld) exceeds batch size (%lld), increase batch size and re-run\n", + __func__, (long long int) inp.size(), (long long int) n_batch); + return 1; } inputs.push_back(inp); } From d0541094a1b93123a5b6706503a809977d2da0d1 Mon Sep 17 00:00:00 2001 From: Minsoo Cheong <54794500+mscheong01@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:25:16 +0900 Subject: [PATCH 2/4] Update examples/embedding/embedding.cpp Co-authored-by: Xuan Son Nguyen --- examples/embedding/embedding.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index f2dc38a817118..9aede7fadfe31 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -61,7 +61,7 @@ int main(int argc, char ** argv) { } params.embedding = true; - // For BERT models, batch size must be equal to ubatch size + // For non-causal models, batch size must be equal to ubatch size params.n_ubatch = params.n_batch; print_build_info(); From ea753ede90a86a0699f65878cc8e2020ff5eabb8 Mon Sep 17 00:00:00 2001 From: Minsoo Cheong Date: Tue, 26 Mar 2024 10:56:18 +0900 Subject: [PATCH 3/4] use %ld instead of %lld --- examples/embedding/embedding.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index f2dc38a817118..73bec610efd81 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -116,8 +116,8 @@ int main(int argc, char ** argv) { for (const auto & prompt : prompts) { auto inp = ::llama_tokenize(ctx, prompt, true, false); if (inp.size() > n_batch) { - fprintf(stderr, "%s: error: number of tokens in input line (%lld) exceeds batch size (%lld), increase batch size and re-run\n", - __func__, (long long int) inp.size(), (long long int) n_batch); + fprintf(stderr, "%s: error: number of tokens in input line (%ld) exceeds batch size (%ld), increase batch size and re-run\n", + __func__, inp.size(), n_batch); return 1; } inputs.push_back(inp); From 2258098c498d21608d65465d9071aa77d9ed967c Mon Sep 17 00:00:00 2001 From: Minsoo Cheong Date: Tue, 26 Mar 2024 11:13:14 +0900 Subject: [PATCH 4/4] Revert "use %ld instead of %lld" This reverts commit ea753ede90a86a0699f65878cc8e2020ff5eabb8. --- examples/embedding/embedding.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index b0805fe8259f4..9aede7fadfe31 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -116,8 +116,8 @@ int main(int argc, char ** argv) { for (const auto & prompt : prompts) { auto inp = ::llama_tokenize(ctx, prompt, true, false); if (inp.size() > n_batch) { - fprintf(stderr, "%s: error: number of tokens in input line (%ld) exceeds batch size (%ld), increase batch size and re-run\n", - __func__, inp.size(), n_batch); + fprintf(stderr, "%s: error: number of tokens in input line (%lld) exceeds batch size (%lld), increase batch size and re-run\n", + __func__, (long long int) inp.size(), (long long int) n_batch); return 1; } inputs.push_back(inp);