-
Notifications
You must be signed in to change notification settings - Fork 1
/
train.py
96 lines (77 loc) · 3.46 KB
/
train.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import haiku as hk
import jax.numpy as jnp
import numpy as np
import dreamer.env_wrappers as env_wrappers
import dreamer.models as models
import train_utils as train_utils
from dreamer.blocks import DenseDecoder, Decoder
from dreamer.dreamer import Dreamer
from dreamer.logger import TrainingLogger
from dreamer.replay_buffer import ReplayBuffer
from dreamer.utils import get_mixed_precision_policy
def create_model(config, observation_space):
def model():
_model = models.WorldModel(observation_space, config)
def filter_state(prev_state, prev_action, observation):
return _model(prev_state, prev_action, observation)
def generate_sequence(initial_state, policy,
policy_params, actions=None):
return _model.generate_sequence(initial_state, policy,
policy_params, actions)
def observe_sequence(observations, actions):
return _model.observe_sequence(observations, actions)
def decode(feature):
return _model.decode(feature)
def init(observations, actions):
return _model.observe_sequence(observations, actions)
return init, (filter_state, generate_sequence, observe_sequence,
decode)
return hk.multi_transform(model)
def create_actor(config, action_space):
actor = hk.without_apply_rng(hk.transform(
lambda obs: models.Actor(tuple(config.actor['output_sizes']) +
(2 * np.prod(action_space.shape),),
config.actor['min_stddev'],
config.initialization)(obs))
)
return actor
def create_critic(config):
critic = hk.without_apply_rng(hk.transform(
lambda obs: DenseDecoder(tuple(config.critic['output_sizes']) + (1,),
'normal', config.initialization)(obs)
))
return critic
def make_agent(config, environment, logger):
experience = ReplayBuffer(config.replay['capacity'],
environment.observation_space,
environment.action_space,
config.replay['batch'],
config.replay['sequence_length'],
config.precision,
config.seed)
precision_policy = get_mixed_precision_policy(config.precision)
agent = Dreamer(environment.observation_space,
environment.action_space,
create_model(config, environment.observation_space),
create_actor(config, environment.action_space),
create_critic(config), experience,
logger, config, precision_policy)
return agent
if __name__ == '__main__':
config = train_utils.load_config()
np.random.seed(config.seed)
if not config.jit:
from jax.config import config as jax_config
jax_config.update('jax_disable_jit', True)
if config.precision == 16:
policy = get_mixed_precision_policy(config.precision)
hk.mixed_precision.set_policy(models.WorldModel, policy)
hk.mixed_precision.set_policy(models.Actor, policy)
hk.mixed_precision.set_policy(DenseDecoder, policy)
hk.mixed_precision.set_policy(Decoder, policy.with_output_dtype(
jnp.float32))
environment = env_wrappers.make_env(config.task, config.time_limit,
config.action_repeat, config.seed)
logger = TrainingLogger(config.log_dir)
agent = make_agent(config, environment, logger)
train_utils.train(config, agent, environment, logger)