System information

• OS Platform and Distribution (e.g., Linux Ubuntu 16.04): ubuntu 16.04 LTS
• Ray installed from (source or binary): pip
• Ray version: 0.3.1
• Python version: Python 3.6.3
• Exact command to reproduce:
  import ray
  from ray.rllib import dqn
  ray.init()
  agent = dqn.DQNAgent(env='CartPole-v0', config={'num_workers':2})

Describe the problem

There is a problem when trying to run a dqn agent with multiple workers. The error is in dqn_replay_evaluator.py line 31.

it says 'remote_cls.remote(env_creator, config, logdir)', while it should say 'remote_cls.remote(registry,env_creator, config, logdir)'

However when this is fixed, another problem arises caused by the no updating the global timestep for the remote evaluators/workers. The global time step does not increase and thus the exploration does not decrease.

Source code / logs

Exception Traceback (most recent call last)
in ()
----> 1 agent = dqn.DQNAgent(env='CartPole-v0', config={'num_workers':2})

~/anaconda3/lib/python3.6/site-packages/ray/rllib/agent.py in init(self, config, env, registry, logger_creator)
82 # Agents allow env ids to be passed directly to the constructor.
83 self._env_id = env or config.get("env")
---> 84 Trainable.init(self, config, registry, logger_creator)
85
86 def _setup(self):

~/anaconda3/lib/python3.6/site-packages/ray/tune/trainable.py in init(self, config, registry, logger_creator)
86 self._time_total = 0.0
87 self._timesteps_total = 0
---> 88 self._setup()
89 self._initialize_ok = True
90

~/anaconda3/lib/python3.6/site-packages/ray/rllib/agent.py in _setup(self)
105 # TODO(ekl) setting the graph is unnecessary for PyTorch agents
106 with tf.Graph().as_default():
--> 107 self._init()
108
109 def _init(self):

~/anaconda3/lib/python3.6/site-packages/ray/rllib/dqn/dqn.py in _init(self)
135 else:
136 self.local_evaluator = DQNReplayEvaluator(
--> 137 self.registry, self.env_creator, self.config, self.logdir)
138 # No remote evaluators. If num_workers > 1, the DQNReplayEvaluator
139 # will internally create more workers for parallelism. This means

~/anaconda3/lib/python3.6/site-packages/ray/rllib/dqn/dqn_replay_evaluator.py in init(self, registry, env_creator, config, logdir)
30 self.workers = [
31 remote_cls.remote(env_creator, config, logdir)
---> 32 for _ in range(self.config["num_workers"])]
33 else:
34 self.workers = []

~/anaconda3/lib/python3.6/site-packages/ray/rllib/dqn/dqn_replay_evaluator.py in (.0)
30 self.workers = [
31 remote_cls.remote(env_creator, config, logdir)
---> 32 for _ in range(self.config["num_workers"])]
33 else:
34 self.workers = []

~/anaconda3/lib/python3.6/site-packages/ray/actor.py in remote(cls, *args, **kwargs)
739 if "init" in actor_object._ray_actor_method_names:
740 actor_object._actor_method_call("init", args=args,
--> 741 kwargs=kwargs)
742 else:
743 print("WARNING: this object has no init method.")

~/anaconda3/lib/python3.6/site-packages/ray/actor.py in _actor_method_call(self, method_name, args, kwargs, dependency)
571 if kwargs is None:
572 kwargs = {}
--> 573 args = signature.extend_args(function_signature, args, kwargs)
574
575 # Execute functions locally if Ray is run in PYTHON_MODE

~/anaconda3/lib/python3.6/site-packages/ray/signature.py in extend_args(function_signature, args, kwargs)
203 raise Exception("No value was provided for the argument "
204 "'{}' for the function '{}'."
--> 205 .format(keyword_name, function_name))
206
207 too_many_arguments = (len(args) > len(arg_names) and

Exception: No value was provided for the argument 'logdir' for the function 'init'.