diff options
Diffstat (limited to 'trm/config/arch/transformers_baseline.yaml')
| -rw-r--r-- | trm/config/arch/transformers_baseline.yaml | 18 |
1 files changed, 18 insertions, 0 deletions
diff --git a/trm/config/arch/transformers_baseline.yaml b/trm/config/arch/transformers_baseline.yaml new file mode 100644 index 0000000..cc08ffe --- /dev/null +++ b/trm/config/arch/transformers_baseline.yaml @@ -0,0 +1,18 @@ +name: recursive_reasoning.transformers_baseline@Model_ACTV2 +loss: + name: losses@ACTLossHead + loss_type: stablemax_cross_entropy + +halt_exploration_prob: 0.1 +halt_max_steps: 16 + +H_cycles: 1 # kept for compatibility +H_layers: 8 + +hidden_size: 512 +num_heads: 12 +expansion: 4 + +puzzle_emb_ndim: ${.hidden_size} + +pos_encodings: rope
\ No newline at end of file |