Skip to content

Commit

Permalink
Gradient rope formula with offsets (ggerganov#938)
Browse files Browse the repository at this point in the history 
* Gradient rope formula with offsets

Positive for Solar models
Negative for Llama 1 and 2 models

* Update gpttype_adapter.cpp

Remove L1/L2

* cleanup PR, skip llama models, keep prints behind debug mode

---------

Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
  • Loading branch information
@Nexesenex @LostRuins
Nexesenex and LostRuins authored Jun 25, 2024
1 parent dd5cda0 commit cb2336f
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 10 deletions.
60 changes: 52 additions & 8 deletions gpttype_adapter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -789,16 +789,59 @@ static int GetBatchSize(int desiredBlasBatchSize,FileFormat in_file_format)
}

//this function applies automatic scaling to rope freq base when the desired context exceeds trained context
static float CalcGradientAIRopeFreqBase(float original_rope_base, int n_ctx_train, int n_ctx_desired, bool is_solar)
static float CalcGradientAIRopeFreqBase(float original_rope_base, int n_ctx_train, int n_ctx_desired, GGUFArch model_arch)
{
if(n_ctx_desired <= n_ctx_train || n_ctx_desired <= 2048)
{
return original_rope_base;
}
float ctx_multiplier = (is_solar?8.0f:1.0f);
float chi_ctx_train_value = (n_ctx_train * ctx_multiplier) / 6.28318;
float chi_ctx_value = (n_ctx_desired * ctx_multiplier) / 6.28318;
return powf(original_rope_base, logf(chi_ctx_value) / logf(chi_ctx_train_value));
else
{
float ctx_multiplier = (model_arch==GGUFArch::ARCH_SOLAR?8.0f:1.0f);
float chi_ctx_train_value = (n_ctx_train * ctx_multiplier) / 6.28318;
float chi_ctx_value = (n_ctx_desired * ctx_multiplier) / 6.28318;
float gradient_ai_rope_freq_base_value = powf(original_rope_base, log10f(chi_ctx_value) / log10f(chi_ctx_train_value));

if(debugmode==1)
{
printf("Trained max context length (value:%.d).\n", n_ctx_train);
printf("Desired context length (value:%.d).\n", n_ctx_desired);
printf("Solar context multiplier (value:%.3f).\n", ctx_multiplier);
printf("Chi context train (value:%.3f).\n", chi_ctx_train_value);
printf("Chi chosen context (value:%.3f).\n", chi_ctx_value);
printf("Log Chi context train (value:%.3f).\n", log10f(chi_ctx_train_value));
printf("Log Chi chosen context (value:%.3f).\n", log10f(chi_ctx_value));
printf("RoPE Frequency Base value (value:%.3f).\n", original_rope_base);
printf("RoPE base calculated via Gradient AI formula. (value:%.1f).\n", gradient_ai_rope_freq_base_value);
}

if(model_arch==GGUFArch::ARCH_SOLAR)
{
float extended_rope_positive_offset_value = 1 + ((log10f(chi_ctx_value) - log10f(chi_ctx_train_value)) / ((log10f(chi_ctx_value) * log10f(chi_ctx_train_value)) - (log10f(chi_ctx_value) + log10f(chi_ctx_train_value))));
float rope_freq_base_with_positive_offset = gradient_ai_rope_freq_base_value * extended_rope_positive_offset_value;
if(debugmode==1)
{
printf("Extended RoPE Positive Offset (multiplicator) for Solar based models. (value:%.3f).\n", extended_rope_positive_offset_value);
printf("RoPE base calculated via Gradient AI formula for Solar based models. (value:%.1f).\n", rope_freq_base_with_positive_offset);
}
return rope_freq_base_with_positive_offset;
}
// else if(model_arch==GGUFArch::ARCH_MISTRAL_LLAMA_1_AND_2)
// {
// float extended_rope_negative_offset_value = 1 + ((log10f(chi_ctx_value) - log10f(chi_ctx_train_value)) / (3.14159265358979323846 * 3.14159265358979323846));
// float rope_freq_base_with_negative_offset = gradient_ai_rope_freq_base_value / extended_rope_negative_offset_value;
// if(debugmode==1)
// {
// printf("Extended RoPE Negative Offset (divisor) for Llama 1 and 2 based models. (value:%.3f).\n", extended_rope_negative_offset_value);
// printf("RoPE base calculated via Gradient AI formula for Llama 1 and 2 based models. (value:%.1f).\n", rope_freq_base_with_negative_offset);
// }
// return rope_freq_base_with_negative_offset;
// }
else
{
return gradient_ai_rope_freq_base_value;
}
}
}

ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta in_file_format_meta)
Expand Down Expand Up @@ -850,10 +893,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
else
{
//Set freq base for all, including non GGUF. If we are using GGUF, this will be overwritten with more accurate values later.
rope_freq_base = CalcGradientAIRopeFreqBase(10000.0f,2048,kcpp_params->n_ctx,false);
rope_freq_base = CalcGradientAIRopeFreqBase(10000.0f,2048,kcpp_params->n_ctx, GGUFArch::ARCH_DEFAULT);
if(file_format==FileFormat::GGUF_GENERIC)
{
printf("Using automatic RoPE scaling. If the model has customized RoPE settings, they will be used directly instead!\n");
printf("Using automatic RoPE scaling for GGUF. If the model has custom RoPE settings, they'll be used directly instead!\n");
printf("It means that the RoPE values written above will be replaced by the RoPE values indicated after loading.\n");
}
else
{
Expand Down Expand Up @@ -1099,7 +1143,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
else
{
//Calculate rope_freq_base using the gradientAI formula, solar requires ctx *8 for correct scaling
rope_freq_base = CalcGradientAIRopeFreqBase(llamamodel->hparams.rope_freq_base_train, file_format_meta.n_ctx_train, kcpp_params->n_ctx, file_format_meta.model_architecture==GGUFArch::ARCH_SOLAR);
rope_freq_base = CalcGradientAIRopeFreqBase(llamamodel->hparams.rope_freq_base_train, file_format_meta.n_ctx_train, kcpp_params->n_ctx, file_format_meta.model_architecture);
llama_ctx_params.rope_freq_base = rope_freq_base;
llama_ctx_params.rope_freq_scale = rope_freq_scale;
printf("Automatic RoPE Scaling: Using (scale:%.3f, base:%.1f).\n", rope_freq_scale, rope_freq_base);
Expand Down
8 changes: 7 additions & 1 deletion model_adapter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,10 +306,16 @@ void print_tok_vec(std::vector<float> &embd)
{
fileformatmeta->model_architecture = GGUFArch::ARCH_MAMBA;
}
else if(modelarch=="llama" && freq_base_train==10000.0f && n_tensors==435)
else if(modelarch=="llama" && freq_base_train==10000.0f && (n_tensors==435 || n_tensors==611))
{
fileformatmeta->model_architecture = GGUFArch::ARCH_SOLAR;
}
else if(modelarch=="llama" && freq_base_train==10000.0f)
{
fileformatmeta->model_architecture = GGUFArch::ARCH_MISTRAL_LLAMA_1_AND_2;
}
printf("Arch Category: %d\n",fileformatmeta->model_architecture);

}

gguf_free(ctx);
Expand Down
3 changes: 2 additions & 1 deletion model_adapter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,12 @@ enum FileFormat

enum GGUFArch
{
ARCH_DEFAULT = 0, //used for llama and other generic gguf
ARCH_DEFAULT = 0, //used for llama3 and other generic gguf
ARCH_FALCON = 1,
ARCH_PHI = 2,
ARCH_MAMBA = 3,
ARCH_SOLAR = 4,
ARCH_MISTRAL_LLAMA_1_AND_2 = 5,
};

struct FileFormatExtraMeta
Expand Down

0 comments on commit cb2336f

Please sign in to comment.