Skip to content

Commit

Permalink
compact dynatemp into a single value dynatemp_range. This is a float …
Browse files Browse the repository at this point in the history 
…which represents the allowed deviation from the min and max temperature when using dynatemp. Thus, if we want a value of dynatemp_min=0.3, dynatemp_max=0.5, then we would simply set temperature=0.4 and dynatemp_range=0.1. Functionally dynatemp would operate the same, but it would simplify usage and make it a single easy to adjust value.
  • Loading branch information
@LostRuins
LostRuins committed Jan 5, 2024
1 parent 79aeadf commit 41d04bc
Show file tree
Hide file tree
Showing 8 changed files with 38 additions and 89 deletions.
4 changes: 1 addition & 3 deletions common/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,7 @@ struct gpt_params {
float mirostat_eta = 0.10f; // learning rate

// DynaTemp!
bool dynatemp = false; // enable DynaTemp
float min_temp = 0.00f; // minimum temperature
float max_temp = 2.00f; // maximum temperature
float dynatemp_range = 0.0f; // enables DynaTemp if greater than 0. dynatemp_min = temperature - dt_range, dynatemp_max = temperature + dt_range

// // sampling parameters
struct llama_sampling_params sparams;
Expand Down
4 changes: 1 addition & 3 deletions common/sampling.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,7 @@ typedef struct llama_sampling_params {
int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
float mirostat_tau = 5.00f; // target entropy
float mirostat_eta = 0.10f; // learning rate
bool dynatemp = false; // dynamic temperature
float min_temp = 0.00f; // minimum temperature
float max_temp = 2.00f; // maximum temperature
bool dynatemp_range = 0.00f; // dynamic temperature range
bool penalize_nl = true; // consider newlines as a repeatable token
std::string samplers_sequence = "kfypmt"; // top_k, tail_free, typical_p, top_p, min_p, temp

Expand Down
4 changes: 1 addition & 3 deletions expose.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,7 @@ struct generation_inputs
const char * grammar;
const bool grammar_retain_state;
const bool quiet = false;
const bool dynatemp = false;
const float min_temp;
const float max_temp;
const float dynatemp_range = 0.0f;
const logit_bias logit_biases[logit_bias_max];

};
Expand Down
21 changes: 11 additions & 10 deletions gpttype_adapter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ void sample_grammar(FileFormat file_format, int32_t n_vocab, llama_token_data_ar
}

int SampleLogits(const float * logits, int n_ctx, int n_vocab, int rep_pen_range, float rep_pen, float presence_penalty, float top_k, float top_a, float top_p, float min_p, float typical_p, float tfs, float temp, std::mt19937 & rng,
int mirostat, float mirostat_tau, float mirostat_eta, const std::vector<samplers> & sampler_order, llama_grammar * grammar, bool dynatemp, float min_temp, float max_temp)
int mirostat, float mirostat_tau, float mirostat_eta, const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dynatemp_range)
{
int id = 0;
std::vector<llama_token_data> candidates;
Expand Down Expand Up @@ -540,9 +540,14 @@ int mirostat, float mirostat_tau, float mirostat_eta, const std::vector<samplers
llama_sample_typical(nullptr, &candidates_p, typical_p,1);
break;
case KCPP_SAMPLER_TEMP:
if (dynatemp)
if (dynatemp_range>0)
{
llama_sample_entropy(nullptr, &candidates_p, temp, min_temp, max_temp);
float dynatemp_min = temp - dynatemp_range;
float dynatemp_max = temp + dynatemp_range;
//do not allow negative values
dynatemp_min = dynatemp_min<0?0:dynatemp_min;
dynatemp_max = dynatemp_max<0?0:dynatemp_max;
llama_sample_entropy(nullptr, &candidates_p, temp, dynatemp_min, dynatemp_max);
}
else
{
Expand Down Expand Up @@ -1502,9 +1507,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
kcpp_params->mirostat = inputs.mirostat;
kcpp_params->mirostat_eta = inputs.mirostat_eta;
kcpp_params->mirostat_tau = inputs.mirostat_tau;
kcpp_params->dynatemp = inputs.dynatemp;
kcpp_params->min_temp = inputs.min_temp;
kcpp_params->max_temp = inputs.max_temp;
kcpp_params->dynatemp_range = inputs.dynatemp_range;
kcpp_params->n_ctx = inputs.max_context_length;
kcpp_params->n_batch = n_batch;
kcpp_params->n_threads = n_threads;
Expand Down Expand Up @@ -1900,9 +1903,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
const float presence_penalty = kcpp_params->presence_penalty;
const float typical_p = kcpp_params->typical_p;
const float tfs_z = kcpp_params->tfs_z;
const float dynatemp = kcpp_params->dynatemp;
const float min_temp = kcpp_params->min_temp;
const float max_temp = kcpp_params->max_temp;
const float dynatemp_range = kcpp_params->dynatemp_range;

if (!startedsampling)
{
Expand Down Expand Up @@ -1958,7 +1959,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o

id = SampleLogits(logitsPtr, nctx, n_vocab, last_n_size, repeat_penalty, presence_penalty,
top_k, top_a, top_p, min_p, typical_p, tfs_z, temp, rng,
kcpp_params->mirostat, kcpp_params->mirostat_tau, kcpp_params->mirostat_eta, sampler_order, grammar, dynatemp, min_temp, max_temp);
kcpp_params->mirostat, kcpp_params->mirostat_tau, kcpp_params->mirostat_eta, sampler_order, grammar, dynatemp_range);

if (grammar != nullptr) {
grammar_accept_token(file_format, n_vocab, grammar, id);
Expand Down
15 changes: 3 additions & 12 deletions kcpp_docs.embd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,18 +139,9 @@
"description": "If true, prevents the EOS token from being generated (Ban EOS). For unbantokens, set this to false.",
"type": "boolean"
},
"dynatemp": {
"default": false,
"description": "If true, uses dynamic temperature. If false, uses static temperature.",
"type": "boolean"
},
"min_temp": {
"description": "Dynatemp Minimum temperature value.",
"exclusiveMinimum": 0,
"type": "number"
},
"max_temp": {
"description": "Maximum temperature value.",
"dynatemp_range": {
"default": 0,
"description": "If greater than 0, uses dynamic temperature. Dynamic temperature range will be between Temp+Range and Temp-Range. If less or equal to 0 , uses static temperature.",
"exclusiveMinimum": 0,
"type": "number"
},
Expand Down
Loading

0 comments on commit 41d04bc

Please sign in to comment.