Hebrew_GPT2_tiny.json

{
    "n_head": 12,
    "n_vocab": 50257,
    "embed_dropout": 0.1,
    "lr": 0.0002,
    "lr_decay": "cosine",
    "warmup_steps": 3000,
    "beta1": 0.9,
    "beta2": 0.95,
    "epsilon": 1e-8,
    "opt_name": "adam",
    "weight_decay": 0,
    "train_batch_size": 250,
    "tokens_per_mb_per_replica": 2048,
    "attn_dropout": 0.1,
    "train_steps": 1000000,
    "lr_decay_end": 300000,
    "eval_steps": 0,
    "predict_steps": 1,
    "res_dropout": 0.1,
    "eval_batch_size": 64,
    "predict_batch_size": 1,
    "iterations": 250,
    "n_embd": 768,
    "datasets": [["Hebrew", null, null, null], ["Hebrew2", null, null, null]],
    "model": "GPT",
    "model_path": "gs://dadler_gptneo_europe-west4/models/Hebrew_GPT2_tiny",
    "n_ctx": 1024,
    "n_layer": 6,
    "scale_by_depth": true,
    "scale_by_in": false,
    "attention_types" :  [[["global"],6]],
    "activation_function": "gelu",
    "mesh_shape": "x:4,y:2",
    "layout": "intermediate_expanded:x,heads:x,vocab:n_vocab,memory_length:y,embd:y",
    "recompute_grad": false,
    "gradient_clipping": 1.0,
    "precision": "bfloat16"
}