-
Notifications
You must be signed in to change notification settings - Fork 31
/
xl_net_python.yaml
79 lines (72 loc) · 1.66 KB
/
xl_net_python.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
experiment_setup:
executable: 'code_transformer/experiments/xl_net/code_summarization.py'
data_setup:
language: 'python'
use_validation: True
num_sub_tokens: 5
num_subtokens_output: 6
use_no_punctuation: True
use_pointer_network: True
data_transforms:
max_distance_mask: None
relative_distances: None
distance_binning:
type: 'exponential'
growth_factor: 1.3
n_fixed_bins: 9
transfer_learning:
use_pretrained_model: False
model_type: 'xl_net_lm'
run_id: 4
snapshot_iteration: 'latest'
cpu: False
freeze_encoder_layers: None
model:
with_cuda: True
label_smoothing: 0.1
lm_encoder:
subtokens_per_token: 5
num_languages: None
input_nonlinearity: 'tanh'
transformer:
d_model: 1024
n_layer: 3
n_head: 8
d_inner: 2048
ff_activation: 'gelu'
dropout: 0.2
mem_len: 1024
lm_decoder:
output_nonlinearity: None
n_layers: 1
decoder_dropout: 0
decoder_nhead: 8
decoder_dim_feedforward: 2048
decoder_activation: 'gelu'
use_teacher_forcing: True
pointer_attention_type: 'additive'
use_pointer_query_linear: False
use_pointer_query_self_attention: False
attend_cls_token: False
optimizer:
optimizer: 'Adam'
learning_rate: 8e-5
reg_scale: 3e-5
training:
random_seed: 456
batch_size: 8
simulated_batch_size: 128
simulated_batch_size_valid: 1280
accumulate_tokens_batch: False
validate_every: 100
persistent_snapshot_every: 10000
early_stopping_patience: 20
max_validation_samples: 50000
metrics:
- top1_accuracy
- top5_accuracy
- non_trivial_accuracy
- precision
- recall
- f1_score
- micro_f1_score