[XRay] Custom label support #1869
Description
I'm trying this:
First node:
ray start --head --use-raylet --resources='{"Actor": 1}'
Second node:
ray start --redis-address 172.31.23.99:61864 --use-raylet --resources='{"Actor": 1}'
And then run the following code:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import ray
@ray.remote(resources={'Actor': 1})
class Worker(object):
def __init__(self):
pass
ray.init(redis_address="172.31.23.99:61864", use_raylet=True)
w = Worker.remote()
It will crash with
/home/ubuntu/ray/src/ray/raylet/scheduling_policy.cc:57 Check failed: !client_keys.empty()
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet[0x449970]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN3ray6raylet16SchedulingPolicy8ScheduleERKSt13unordered_mapINS_8UniqueIDENS0_19SchedulingResourcesENS_14UniqueIDHasherESt8equal_toIS3_ESaISt4pairIKS3_S4_EEERS9_RKSt6vectorIS3_SaIS3_EE+0x5d7)[0x49c807]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN3ray6raylet11NodeManager13ScheduleTasksEv+0x68)[0x49ffc8]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN3ray6raylet11NodeManager9QueueTaskERKNS0_4TaskE+0x86)[0x4a0846]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN3ray6raylet11NodeManager10SubmitTaskERKNS0_4TaskERKNS0_7LineageE+0x1ab)[0x4a0a8b]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN3ray6raylet11NodeManager20ProcessClientMessageESt10shared_ptrINS_16ClientConnectionIN5boost4asio5local15stream_protocolEEEElPKh+0x316)[0x4a1006]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet[0x456db1]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN3ray16ClientConnectionIN5boost4asio5local15stream_protocolEE14ProcessMessageERKNS1_6system10error_codeE+0x97)[0x4860b7]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN5boost4asio6detail23reactive_socket_recv_opINS0_17mutable_buffers_1ENS1_7read_opINS0_19basic_stream_socketINS0_5local15stream_protocolENS0_21stream_socket_serviceIS7_EEEES3_NS1_14transfer_all_tENS_3_bi6bind_tIvNS_4_mfi3mf1IvN3ray16ClientConnectionIS7_EERKNS_6system10error_codeEEENSC_5list2INSC_5valueISt10shared_ptrISI_EEEPFNS_3argILi1EEEvEEEEEEEE11do_completeEPNS1_15task_io_serviceEPNS1_25task_io_service_operationESM_m+0x14d)[0x48640d]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_ZN5boost4asio6detail15task_io_service3runERNS_6system10error_codeE+0x407)[0x4523f7]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(main+0xd62)[0x44aac2]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7fc254aea830]
/home/ubuntu/ray/python/ray/core/src/ray/raylet/raylet(_start+0x29)[0x44e499]