Flaky test_dashboard #4697
Description
distributed/cli/tests/test_dask_scheduler.py::test_dashboard has started sporadically failing over the past few days (for example, here). See the full traceback below.
Details:
2021-04-13T01:44:41.4151490Z ________________________________ test_dashboard ________________________________
2021-04-13T01:44:41.4151858Z
2021-04-13T01:44:41.4152669Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4153343Z
2021-04-13T01:44:41.4153710Z def _new_conn(self):
2021-04-13T01:44:41.4215365Z """Establish a socket connection and set nodelay settings on it.
2021-04-13T01:44:41.4216356Z
2021-04-13T01:44:41.4216841Z :return: New socket connection.
2021-04-13T01:44:41.4217283Z """
2021-04-13T01:44:41.4217640Z extra_kw = {}
2021-04-13T01:44:41.4218473Z if self.source_address:
2021-04-13T01:44:41.4219094Z extra_kw["source_address"] = self.source_address
2021-04-13T01:44:41.4219852Z
2021-04-13T01:44:41.4220432Z if self.socket_options:
2021-04-13T01:44:41.4221063Z extra_kw["socket_options"] = self.socket_options
2021-04-13T01:44:41.4221559Z
2021-04-13T01:44:41.4223112Z try:
2021-04-13T01:44:41.4223642Z conn = connection.create_connection(
2021-04-13T01:44:41.4224414Z > (self._dns_host, self.port), self.timeout, **extra_kw
2021-04-13T01:44:41.4224953Z )
2021-04-13T01:44:41.4225198Z
2021-04-13T01:44:41.4297569Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connection.py:170:
2021-04-13T01:44:41.4298502Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4298817Z
2021-04-13T01:44:41.4299611Z address = ('fv-az212-648', 8787), timeout = None, source_address = None
2021-04-13T01:44:41.4300243Z socket_options = [(6, 1, 1)]
2021-04-13T01:44:41.4300534Z
2021-04-13T01:44:41.4300952Z def create_connection(
2021-04-13T01:44:41.4301410Z address,
2021-04-13T01:44:41.4301940Z timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
2021-04-13T01:44:41.4302499Z source_address=None,
2021-04-13T01:44:41.4302981Z socket_options=None,
2021-04-13T01:44:41.4303369Z ):
2021-04-13T01:44:41.4304086Z """Connect to *address* and return the socket object.
2021-04-13T01:44:41.4304600Z
2021-04-13T01:44:41.4305370Z Convenience function. Connect to *address* (a 2-tuple ``(host,
2021-04-13T01:44:41.4306178Z port)``) and return the socket object. Passing the optional
2021-04-13T01:44:41.4306960Z *timeout* parameter will set the timeout on the socket instance
2021-04-13T01:44:41.4307793Z before attempting to connect. If no *timeout* is supplied, the
2021-04-13T01:44:41.4308772Z global default timeout setting returned by :func:`socket.getdefaulttimeout`
2021-04-13T01:44:41.4309737Z is used. If *source_address* is set it must be a tuple of (host, port)
2021-04-13T01:44:41.4310561Z for the socket to bind as a source address before making the connection.
2021-04-13T01:44:41.4314257Z An host of '' or port 0 tells the OS to use the default.
2021-04-13T01:44:41.4315785Z """
2021-04-13T01:44:41.4316163Z
2021-04-13T01:44:41.4316561Z host, port = address
2021-04-13T01:44:41.4317062Z if host.startswith("["):
2021-04-13T01:44:41.4317556Z host = host.strip("[]")
2021-04-13T01:44:41.4317998Z err = None
2021-04-13T01:44:41.4318340Z
2021-04-13T01:44:41.4318942Z # Using the value from allowed_gai_family() in the context of getaddrinfo lets
2021-04-13T01:44:41.4319847Z # us select whether to work with IPv4 DNS records, IPv6 records, or both.
2021-04-13T01:44:41.4320708Z # The original create_connection function always returns all records.
2021-04-13T01:44:41.4321413Z family = allowed_gai_family()
2021-04-13T01:44:41.4321818Z
2021-04-13T01:44:41.4322150Z try:
2021-04-13T01:44:41.4322559Z host.encode("idna")
2021-04-13T01:44:41.4323077Z except UnicodeError:
2021-04-13T01:44:41.4323587Z return six.raise_from(
2021-04-13T01:44:41.4324583Z LocationParseError(u"'%s', label empty or too long" % host), None
2021-04-13T01:44:41.4325182Z )
2021-04-13T01:44:41.4325514Z
2021-04-13T01:44:41.4326163Z for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
2021-04-13T01:44:41.4326977Z af, socktype, proto, canonname, sa = res
2021-04-13T01:44:41.4327486Z sock = None
2021-04-13T01:44:41.4327847Z try:
2021-04-13T01:44:41.4328387Z sock = socket.socket(af, socktype, proto)
2021-04-13T01:44:41.4328879Z
2021-04-13T01:44:41.4329456Z # If provided, set socket level options before connecting.
2021-04-13T01:44:41.4330182Z _set_socket_options(sock, socket_options)
2021-04-13T01:44:41.4331368Z
2021-04-13T01:44:41.4332035Z if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
2021-04-13T01:44:41.4332893Z sock.settimeout(timeout)
2021-04-13T01:44:41.4333426Z if source_address:
2021-04-13T01:44:41.4333947Z sock.bind(source_address)
2021-04-13T01:44:41.4334458Z sock.connect(sa)
2021-04-13T01:44:41.4334915Z return sock
2021-04-13T01:44:41.4335275Z
2021-04-13T01:44:41.4335724Z except socket.error as e:
2021-04-13T01:44:41.4336229Z err = e
2021-04-13T01:44:41.4336635Z if sock is not None:
2021-04-13T01:44:41.4337089Z sock.close()
2021-04-13T01:44:41.4337500Z sock = None
2021-04-13T01:44:41.4337862Z
2021-04-13T01:44:41.4338225Z if err is not None:
2021-04-13T01:44:41.4338647Z > raise err
2021-04-13T01:44:41.4338901Z
2021-04-13T01:44:41.4340018Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/util/connection.py:96:
2021-04-13T01:44:41.4340882Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4341180Z
2021-04-13T01:44:41.4341922Z address = ('fv-az212-648', 8787), timeout = None, source_address = None
2021-04-13T01:44:41.4342646Z socket_options = [(6, 1, 1)]
2021-04-13T01:44:41.4342945Z
2021-04-13T01:44:41.4343343Z def create_connection(
2021-04-13T01:44:41.4343800Z address,
2021-04-13T01:44:41.4344311Z timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
2021-04-13T01:44:41.4344876Z source_address=None,
2021-04-13T01:44:41.4345344Z socket_options=None,
2021-04-13T01:44:41.4345747Z ):
2021-04-13T01:44:41.4346456Z """Connect to *address* and return the socket object.
2021-04-13T01:44:41.4347360Z
2021-04-13T01:44:41.4348333Z Convenience function. Connect to *address* (a 2-tuple ``(host,
2021-04-13T01:44:41.4349148Z port)``) and return the socket object. Passing the optional
2021-04-13T01:44:41.4349930Z *timeout* parameter will set the timeout on the socket instance
2021-04-13T01:44:41.4350739Z before attempting to connect. If no *timeout* is supplied, the
2021-04-13T01:44:41.4351721Z global default timeout setting returned by :func:`socket.getdefaulttimeout`
2021-04-13T01:44:41.4352662Z is used. If *source_address* is set it must be a tuple of (host, port)
2021-04-13T01:44:41.4353475Z for the socket to bind as a source address before making the connection.
2021-04-13T01:44:41.4354417Z An host of '' or port 0 tells the OS to use the default.
2021-04-13T01:44:41.4354919Z """
2021-04-13T01:44:41.4355232Z
2021-04-13T01:44:41.4355624Z host, port = address
2021-04-13T01:44:41.4356103Z if host.startswith("["):
2021-04-13T01:44:41.4356610Z host = host.strip("[]")
2021-04-13T01:44:41.4357031Z err = None
2021-04-13T01:44:41.4357385Z
2021-04-13T01:44:41.4357987Z # Using the value from allowed_gai_family() in the context of getaddrinfo lets
2021-04-13T01:44:41.4358870Z # us select whether to work with IPv4 DNS records, IPv6 records, or both.
2021-04-13T01:44:41.4359730Z # The original create_connection function always returns all records.
2021-04-13T01:44:41.4360439Z family = allowed_gai_family()
2021-04-13T01:44:41.4360847Z
2021-04-13T01:44:41.4361161Z try:
2021-04-13T01:44:41.4361587Z host.encode("idna")
2021-04-13T01:44:41.4362089Z except UnicodeError:
2021-04-13T01:44:41.4362611Z return six.raise_from(
2021-04-13T01:44:41.4363495Z LocationParseError(u"'%s', label empty or too long" % host), None
2021-04-13T01:44:41.4364103Z )
2021-04-13T01:44:41.4364415Z
2021-04-13T01:44:41.4369159Z for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
2021-04-13T01:44:41.4369971Z af, socktype, proto, canonname, sa = res
2021-04-13T01:44:41.4370500Z sock = None
2021-04-13T01:44:41.4370998Z try:
2021-04-13T01:44:41.4371541Z sock = socket.socket(af, socktype, proto)
2021-04-13T01:44:41.4372036Z
2021-04-13T01:44:41.4372779Z # If provided, set socket level options before connecting.
2021-04-13T01:44:41.4373551Z _set_socket_options(sock, socket_options)
2021-04-13T01:44:41.4374019Z
2021-04-13T01:44:41.4374542Z if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
2021-04-13T01:44:41.4375185Z sock.settimeout(timeout)
2021-04-13T01:44:41.4375726Z if source_address:
2021-04-13T01:44:41.4376229Z sock.bind(source_address)
2021-04-13T01:44:41.4376749Z > sock.connect(sa)
2021-04-13T01:44:41.4377351Z E TimeoutError: [Errno 110] Connection timed out
2021-04-13T01:44:41.4377784Z
2021-04-13T01:44:41.4379027Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/util/connection.py:86: TimeoutError
2021-04-13T01:44:41.4379815Z
2021-04-13T01:44:41.4380434Z During handling of the above exception, another exception occurred:
2021-04-13T01:44:41.4380949Z
2021-04-13T01:44:41.4381891Z self = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f6e99e3a090>
2021-04-13T01:44:41.4383229Z method = 'GET', url = '/status/', body = None
2021-04-13T01:44:41.4384473Z headers = {'User-Agent': 'python-requests/2.25.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
2021-04-13T01:44:41.4385614Z retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
2021-04-13T01:44:41.4386387Z redirect = False, assert_same_host = False
2021-04-13T01:44:41.4387109Z timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
2021-04-13T01:44:41.4387890Z release_conn = False, chunked = False, body_pos = None
2021-04-13T01:44:41.4388821Z response_kw = {'decode_content': False, 'preload_content': False}
2021-04-13T01:44:41.4390283Z parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/status/', query=None, fragment=None)
2021-04-13T01:44:41.4391233Z destination_scheme = None, conn = None, release_this_conn = True
2021-04-13T01:44:41.4391999Z http_tunnel_required = False, err = None, clean_exit = False
2021-04-13T01:44:41.4392437Z
2021-04-13T01:44:41.4392791Z def urlopen(
2021-04-13T01:44:41.4393184Z self,
2021-04-13T01:44:41.4393543Z method,
2021-04-13T01:44:41.4393893Z url,
2021-04-13T01:44:41.4394262Z body=None,
2021-04-13T01:44:41.4394660Z headers=None,
2021-04-13T01:44:41.4395090Z retries=None,
2021-04-13T01:44:41.4395515Z redirect=True,
2021-04-13T01:44:41.4395983Z assert_same_host=True,
2021-04-13T01:44:41.4396440Z timeout=_Default,
2021-04-13T01:44:41.4396909Z pool_timeout=None,
2021-04-13T01:44:41.4397356Z release_conn=None,
2021-04-13T01:44:41.4397806Z chunked=False,
2021-04-13T01:44:41.4398215Z body_pos=None,
2021-04-13T01:44:41.4398923Z **response_kw
2021-04-13T01:44:41.4399544Z ):
2021-04-13T01:44:41.4399857Z """
2021-04-13T01:44:41.4400473Z Get a connection from the pool and perform an HTTP request. This is the
2021-04-13T01:44:41.4401582Z lowest level call for making a request, so you'll need to specify all
2021-04-13T01:44:41.4402230Z the raw details.
2021-04-13T01:44:41.4402595Z
2021-04-13T01:44:41.4402962Z .. note::
2021-04-13T01:44:41.4403290Z
2021-04-13T01:44:41.4404110Z More commonly, it's appropriate to use a convenience method provided
2021-04-13T01:44:41.4404951Z by :class:`.RequestMethods`, such as :meth:`request`.
2021-04-13T01:44:41.4405490Z
2021-04-13T01:44:41.4405826Z .. note::
2021-04-13T01:44:41.4406154Z
2021-04-13T01:44:41.4406663Z `release_conn` will only behave as expected if
2021-04-13T01:44:41.4407345Z `preload_content=False` because we want to make
2021-04-13T01:44:41.4408381Z `preload_content=False` the default behaviour someday soon without
2021-04-13T01:44:41.4409138Z breaking backwards compatibility.
2021-04-13T01:44:41.4409634Z
2021-04-13T01:44:41.4409996Z :param method:
2021-04-13T01:44:41.4410568Z HTTP request method (such as GET, POST, PUT, etc.)
2021-04-13T01:44:41.4411077Z
2021-04-13T01:44:41.4411435Z :param url:
2021-04-13T01:44:41.4411912Z The URL to perform the request on.
2021-04-13T01:44:41.4412365Z
2021-04-13T01:44:41.4412858Z :param body:
2021-04-13T01:44:41.4413452Z Data to send in the request body, either :class:`str`, :class:`bytes`,
2021-04-13T01:44:41.4414479Z an iterable of :class:`str`/:class:`bytes`, or a file-like object.
2021-04-13T01:44:41.4415011Z
2021-04-13T01:44:41.4415389Z :param headers:
2021-04-13T01:44:41.4416176Z Dictionary of custom headers to send, such as User-Agent,
2021-04-13T01:44:41.4417220Z If-None-Match, etc. If None, pool headers are used. If provided,
2021-04-13T01:44:41.4418290Z these headers completely replace any pool-specific headers.
2021-04-13T01:44:41.4418907Z
2021-04-13T01:44:41.4419277Z :param retries:
2021-04-13T01:44:41.4419884Z Configure the number of retries to allow before raising a
2021-04-13T01:44:41.4420992Z :class:`~urllib3.exceptions.MaxRetryError` exception.
2021-04-13T01:44:41.4421722Z
2021-04-13T01:44:41.4422253Z Pass ``None`` to retry until you receive a response. Pass a
2021-04-13T01:44:41.4423362Z :class:`~urllib3.util.retry.Retry` object for fine-grained control
2021-04-13T01:44:41.4424177Z over different types of retries.
2021-04-13T01:44:41.4424910Z Pass an integer number to retry connection errors that many times,
2021-04-13T01:44:41.4425681Z but no other types of errors. Pass zero to never retry.
2021-04-13T01:44:41.4426189Z
2021-04-13T01:44:41.4426776Z If ``False``, then retries are disabled and any exception is raised
2021-04-13T01:44:41.4427791Z immediately. Also, instead of raising a MaxRetryError on redirects,
2021-04-13T01:44:41.4428597Z the redirect response will be returned.
2021-04-13T01:44:41.4429081Z
2021-04-13T01:44:41.4429749Z :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
2021-04-13T01:44:41.4430384Z
2021-04-13T01:44:41.4430776Z :param redirect:
2021-04-13T01:44:41.4431398Z If True, automatically handle redirects (status codes 301, 302,
2021-04-13T01:44:41.4432199Z 303, 307, 308). Each redirect counts as a retry. Disabling retries
2021-04-13T01:44:41.4432833Z will disable redirect, too.
2021-04-13T01:44:41.4433267Z
2021-04-13T01:44:41.4433664Z :param assert_same_host:
2021-04-13T01:44:41.4434292Z If ``True``, will make sure that the host of the pool requests is
2021-04-13T01:44:41.4435135Z consistent else will raise HostChangedError. When ``False``, you can
2021-04-13T01:44:41.4436008Z use the pool on an HTTP proxy and request foreign hosts.
2021-04-13T01:44:41.4436520Z
2021-04-13T01:44:41.4436882Z :param timeout:
2021-04-13T01:44:41.4437509Z If specified, overrides the default timeout for this one
2021-04-13T01:44:41.4438266Z request. It may be a float (in seconds) or an instance of
2021-04-13T01:44:41.4438963Z :class:`urllib3.util.Timeout`.
2021-04-13T01:44:41.4439435Z
2021-04-13T01:44:41.4439832Z :param pool_timeout:
2021-04-13T01:44:41.4440430Z If set and the pool is set to block=True, then this method will
2021-04-13T01:44:41.4441232Z block for ``pool_timeout`` seconds and raise EmptyPoolError if no
2021-04-13T01:44:41.4442020Z connection is available within the time period.
2021-04-13T01:44:41.4442547Z
2021-04-13T01:44:41.4442929Z :param release_conn:
2021-04-13T01:44:41.4443676Z If False, then the urlopen call will not release the connection
2021-04-13T01:44:41.4444485Z back into the pool once a response is received (but will release if
2021-04-13T01:44:41.4445294Z you read the entire contents of the response such as when
2021-04-13T01:44:41.4446291Z `preload_content=True`). This is useful if you're not preloading
2021-04-13T01:44:41.4447265Z the response's content immediately. You will need to call
2021-04-13T01:44:41.4448052Z ``r.release_conn()`` on the response ``r`` to return the connection
2021-04-13T01:44:41.4448792Z back into the pool. If None, it takes the value of
2021-04-13T01:44:41.4449618Z ``response_kw.get('preload_content', True)``.
2021-04-13T01:44:41.4450103Z
2021-04-13T01:44:41.4468194Z :param chunked:
2021-04-13T01:44:41.4468826Z If True, urllib3 will send the body using chunked transfer
2021-04-13T01:44:41.4469640Z encoding. Otherwise, urllib3 will send the body using the standard
2021-04-13T01:44:41.4470892Z content-length form. Defaults to False.
2021-04-13T01:44:41.4471398Z
2021-04-13T01:44:41.4471791Z :param int body_pos:
2021-04-13T01:44:41.4472617Z Position to seek to in file-like body in the event of a retry or
2021-04-13T01:44:41.4473835Z redirect. Typically this won't need to be set because urllib3 will
2021-04-13T01:44:41.4474743Z auto-populate the value when needed.
2021-04-13T01:44:41.4475229Z
2021-04-13T01:44:41.4475636Z :param \\**response_kw:
2021-04-13T01:44:41.4476203Z Additional parameters are passed to
2021-04-13T01:44:41.4477143Z :meth:`urllib3.response.HTTPResponse.from_httplib`
2021-04-13T01:44:41.4477991Z """
2021-04-13T01:44:41.4478322Z
2021-04-13T01:44:41.4478724Z parsed_url = parse_url(url)
2021-04-13T01:44:41.4479433Z destination_scheme = parsed_url.scheme
2021-04-13T01:44:41.4479916Z
2021-04-13T01:44:41.4480313Z if headers is None:
2021-04-13T01:44:41.4480800Z headers = self.headers
2021-04-13T01:44:41.4481210Z
2021-04-13T01:44:41.4481678Z if not isinstance(retries, Retry):
2021-04-13T01:44:41.4482471Z retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
2021-04-13T01:44:41.4483275Z
2021-04-13T01:44:41.4483671Z if release_conn is None:
2021-04-13T01:44:41.4484306Z release_conn = response_kw.get("preload_content", True)
2021-04-13T01:44:41.4484828Z
2021-04-13T01:44:41.4485186Z # Check host
2021-04-13T01:44:41.4485706Z if assert_same_host and not self.is_same_host(url):
2021-04-13T01:44:41.4486418Z raise HostChangedError(self, url, retries)
2021-04-13T01:44:41.4487069Z
2021-04-13T01:44:41.4487833Z # Ensure that the URL we're connecting to is properly encoded
2021-04-13T01:44:41.4488471Z if url.startswith("/"):
2021-04-13T01:44:41.4489052Z url = six.ensure_str(_encode_target(url))
2021-04-13T01:44:41.4489544Z else:
2021-04-13T01:44:41.4490020Z url = six.ensure_str(parsed_url.url)
2021-04-13T01:44:41.4490485Z
2021-04-13T01:44:41.4490821Z conn = None
2021-04-13T01:44:41.4491175Z
2021-04-13T01:44:41.4491672Z # Track whether `conn` needs to be released before
2021-04-13T01:44:41.4492480Z # returning/raising/recursing. Update this variable if necessary, and
2021-04-13T01:44:41.4493662Z # leave `release_conn` constant throughout the function. That way, if
2021-04-13T01:44:41.4494529Z # the function recurses, the original value of `release_conn` will be
2021-04-13T01:44:41.4495383Z # passed down into the recursive call, and its value will be respected.
2021-04-13T01:44:41.4495977Z #
2021-04-13T01:44:41.4496384Z # See issue #651 [1] for details.
2021-04-13T01:44:41.4496803Z #
2021-04-13T01:44:41.4497484Z # [1] <https://github.com/urllib3/urllib3/issues/651>
2021-04-13T01:44:41.4498158Z release_this_conn = release_conn
2021-04-13T01:44:41.4498748Z
2021-04-13T01:44:41.4499299Z http_tunnel_required = connection_requires_http_tunnel(
2021-04-13T01:44:41.4500100Z self.proxy, self.proxy_config, destination_scheme
2021-04-13T01:44:41.4500653Z )
2021-04-13T01:44:41.4500973Z
2021-04-13T01:44:41.4501548Z # Merge the proxy headers. Only done when not using HTTP CONNECT. We
2021-04-13T01:44:41.4502390Z # have to copy the headers dict so we can safely change it without those
2021-04-13T01:44:41.4503365Z # changes being reflected in anyone else's copy.
2021-04-13T01:44:41.4503989Z if not http_tunnel_required:
2021-04-13T01:44:41.4504525Z headers = headers.copy()
2021-04-13T01:44:41.4505166Z headers.update(self.proxy_headers)
2021-04-13T01:44:41.4505660Z
2021-04-13T01:44:41.4506267Z # Must keep the exception bound to a separate variable or else Python 3
2021-04-13T01:44:41.4507059Z # complains about UnboundLocalError.
2021-04-13T01:44:41.4507596Z err = None
2021-04-13T01:44:41.4507952Z
2021-04-13T01:44:41.4508515Z # Keep track of whether we cleanly exited the except block. This
2021-04-13T01:44:41.4509348Z # ensures we do proper cleanup in finally.
2021-04-13T01:44:41.4509881Z clean_exit = False
2021-04-13T01:44:41.4510267Z
2021-04-13T01:44:41.4510814Z # Rewind body position, if needed. Record current position
2021-04-13T01:44:41.4511572Z # for future rewinds in the event of a redirect/retry.
2021-04-13T01:44:41.4512379Z body_pos = set_file_position(body, body_pos)
2021-04-13T01:44:41.4512854Z
2021-04-13T01:44:41.4513172Z try:
2021-04-13T01:44:41.4513673Z # Request a connection from the queue.
2021-04-13T01:44:41.4514293Z timeout_obj = self._get_timeout(timeout)
2021-04-13T01:44:41.4514938Z conn = self._get_conn(timeout=pool_timeout)
2021-04-13T01:44:41.4515401Z
2021-04-13T01:44:41.4515907Z conn.timeout = timeout_obj.connect_timeout
2021-04-13T01:44:41.4516422Z
2021-04-13T01:44:41.4516944Z is_new_proxy_conn = self.proxy is not None and not getattr(
2021-04-13T01:44:41.4517552Z conn, "sock", None
2021-04-13T01:44:41.4517927Z )
2021-04-13T01:44:41.4518437Z if is_new_proxy_conn and http_tunnel_required:
2021-04-13T01:44:41.4519022Z self._prepare_proxy(conn)
2021-04-13T01:44:41.4519453Z
2021-04-13T01:44:41.4519975Z # Make the request on the httplib connection object.
2021-04-13T01:44:41.4520665Z httplib_response = self._make_request(
2021-04-13T01:44:41.4521283Z conn,
2021-04-13T01:44:41.4521668Z method,
2021-04-13T01:44:41.4522027Z url,
2021-04-13T01:44:41.4522447Z timeout=timeout_obj,
2021-04-13T01:44:41.4522906Z body=body,
2021-04-13T01:44:41.4523342Z headers=headers,
2021-04-13T01:44:41.4523818Z > chunked=chunked,
2021-04-13T01:44:41.4524206Z )
2021-04-13T01:44:41.4524447Z
2021-04-13T01:44:41.4525549Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connectionpool.py:706:
2021-04-13T01:44:41.4526806Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4527110Z
2021-04-13T01:44:41.4528217Z self = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f6e99e3a090>
2021-04-13T01:44:41.4529625Z conn = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4530918Z method = 'GET', url = '/status/'
2021-04-13T01:44:41.4531750Z timeout = Timeout(connect=None, read=None, total=None), chunked = False
2021-04-13T01:44:41.4533421Z httplib_request_kw = {'body': None, 'headers': {'User-Agent': 'python-requests/2.25.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}}
2021-04-13T01:44:41.4534771Z timeout_obj = Timeout(connect=None, read=None, total=None)
2021-04-13T01:44:41.4535225Z
2021-04-13T01:44:41.4535602Z def _make_request(
2021-04-13T01:44:41.4536276Z self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
2021-04-13T01:44:41.4536918Z ):
2021-04-13T01:44:41.4537234Z """
2021-04-13T01:44:41.4537847Z Perform a request on a given urllib connection object taken from our
2021-04-13T01:44:41.4538450Z pool.
2021-04-13T01:44:41.4538800Z
2021-04-13T01:44:41.4539147Z :param conn:
2021-04-13T01:44:41.4539701Z a connection from one of our connection pools
2021-04-13T01:44:41.4540185Z
2021-04-13T01:44:41.4540563Z :param timeout:
2021-04-13T01:44:41.4541147Z Socket timeout in seconds for the request. This can be a
2021-04-13T01:44:41.4541903Z float or integer, which will set the same timeout value for
2021-04-13T01:44:41.4542772Z the socket connect and the socket read, or an instance of
2021-04-13T01:44:41.4543865Z :class:`urllib3.util.Timeout`, which gives you more fine-grained
2021-04-13T01:44:41.4544592Z control over your timeouts.
2021-04-13T01:44:41.4545064Z """
2021-04-13T01:44:41.4545470Z self.num_requests += 1
2021-04-13T01:44:41.4545971Z
2021-04-13T01:44:41.4546817Z timeout_obj = self._get_timeout(timeout)
2021-04-13T01:44:41.4547563Z timeout_obj.start_connect()
2021-04-13T01:44:41.4548199Z conn.timeout = timeout_obj.connect_timeout
2021-04-13T01:44:41.4548693Z
2021-04-13T01:44:41.4549190Z # Trigger any extra validation we need to do.
2021-04-13T01:44:41.4549686Z try:
2021-04-13T01:44:41.4550128Z self._validate_conn(conn)
2021-04-13T01:44:41.4550761Z except (SocketTimeout, BaseSSLError) as e:
2021-04-13T01:44:41.4551597Z # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
2021-04-13T01:44:41.4552446Z self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
2021-04-13T01:44:41.4553048Z raise
2021-04-13T01:44:41.4553378Z
2021-04-13T01:44:41.4553951Z # conn.request() calls http.client.*.request, not the method in
2021-04-13T01:44:41.4554814Z # urllib3.request. It also calls makefile (recv) on the socket.
2021-04-13T01:44:41.4555400Z try:
2021-04-13T01:44:41.4555923Z if chunked:
2021-04-13T01:44:41.4556516Z conn.request_chunked(method, url, **httplib_request_kw)
2021-04-13T01:44:41.4557107Z else:
2021-04-13T01:44:41.4557647Z > conn.request(method, url, **httplib_request_kw)
2021-04-13T01:44:41.4558083Z
2021-04-13T01:44:41.4559497Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connectionpool.py:394:
2021-04-13T01:44:41.4560417Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4560863Z
2021-04-13T01:44:41.4561622Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4562719Z method = 'GET', url = '/status/', body = None
2021-04-13T01:44:41.4563937Z headers = {'User-Agent': 'python-requests/2.25.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
2021-04-13T01:44:41.4564714Z
2021-04-13T01:44:41.4565254Z def request(self, method, url, body=None, headers=None):
2021-04-13T01:44:41.4565838Z if headers is None:
2021-04-13T01:44:41.4567692Z headers = {}
2021-04-13T01:44:41.4568319Z else:
2021-04-13T01:44:41.4568873Z # Avoid modifying the headers passed into .request()
2021-04-13T01:44:41.4569516Z headers = headers.copy()
2021-04-13T01:44:41.4570494Z if "user-agent" not in (six.ensure_str(k.lower()) for k in headers):
2021-04-13T01:44:41.4571429Z headers["User-Agent"] = _get_default_user_agent()
2021-04-13T01:44:41.4572430Z > super(HTTPConnection, self).request(method, url, body=body, headers=headers)
2021-04-13T01:44:41.4573111Z
2021-04-13T01:44:41.4574159Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connection.py:234:
2021-04-13T01:44:41.4575010Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4575307Z
2021-04-13T01:44:41.4576058Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4577101Z method = 'GET', url = '/status/', body = None
2021-04-13T01:44:41.4578307Z headers = {'User-Agent': 'python-requests/2.25.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
2021-04-13T01:44:41.4579063Z
2021-04-13T01:44:41.4579585Z def request(self, method, url, body=None, headers={}, *,
2021-04-13T01:44:41.4580187Z encode_chunked=False):
2021-04-13T01:44:41.4580781Z """Send a complete request to the server."""
2021-04-13T01:44:41.4581506Z > self._send_request(method, url, body, headers, encode_chunked)
2021-04-13T01:44:41.4581986Z
2021-04-13T01:44:41.4582838Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/http/client.py:1277:
2021-04-13T01:44:41.4583565Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4583861Z
2021-04-13T01:44:41.4584706Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4585745Z method = 'GET', url = '/status/', body = None
2021-04-13T01:44:41.4586967Z headers = {'User-Agent': 'python-requests/2.25.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
2021-04-13T01:44:41.4587888Z encode_chunked = False
2021-04-13T01:44:41.4588204Z
2021-04-13T01:44:41.4588787Z def _send_request(self, method, url, body, headers, encode_chunked):
2021-04-13T01:44:41.4589849Z # Honor explicitly requested Host: and Accept-Encoding: headers.
2021-04-13T01:44:41.4590672Z header_names = frozenset(k.lower() for k in headers)
2021-04-13T01:44:41.4591206Z skips = {}
2021-04-13T01:44:41.4591791Z if 'host' in header_names:
2021-04-13T01:44:41.4592394Z skips['skip_host'] = 1
2021-04-13T01:44:41.4593106Z if 'accept-encoding' in header_names:
2021-04-13T01:44:41.4593878Z skips['skip_accept_encoding'] = 1
2021-04-13T01:44:41.4594319Z
2021-04-13T01:44:41.4594822Z self.putrequest(method, url, **skips)
2021-04-13T01:44:41.4595305Z
2021-04-13T01:44:41.4595855Z # chunked encoding will happen if HTTP/1.1 is used and either
2021-04-13T01:44:41.4596615Z # the caller passes encode_chunked=True or the following
2021-04-13T01:44:41.4597230Z # conditions hold:
2021-04-13T01:44:41.4597986Z # 1. content-length has not been explicitly set
2021-04-13T01:44:41.4598922Z # 2. the body is a file or iterable, but not a str or bytes-like
2021-04-13T01:44:41.4599916Z # 3. Transfer-Encoding has NOT been explicitly set by the caller
2021-04-13T01:44:41.4600521Z
2021-04-13T01:44:41.4601166Z if 'content-length' not in header_names:
2021-04-13T01:44:41.4603123Z # only chunk body if not explicitly set for backwards
2021-04-13T01:44:41.4604645Z # compatibility, assuming the client code is already handling the
2021-04-13T01:44:41.4605302Z # chunking
2021-04-13T01:44:41.4606589Z if 'transfer-encoding' not in header_names:
2021-04-13T01:44:41.4607612Z # if content-length cannot be automatically determined, fall
2021-04-13T01:44:41.4608357Z # back to chunked encoding
2021-04-13T01:44:41.4608863Z encode_chunked = False
2021-04-13T01:44:41.4609502Z content_length = self._get_content_length(body, method)
2021-04-13T01:44:41.4610139Z if content_length is None:
2021-04-13T01:44:41.4610635Z if body is not None:
2021-04-13T01:44:41.4611149Z if self.debuglevel > 0:
2021-04-13T01:44:41.4612122Z print('Unable to determine size of %r' % body)
2021-04-13T01:44:41.4612894Z encode_chunked = True
2021-04-13T01:44:41.4613795Z self.putheader('Transfer-Encoding', 'chunked')
2021-04-13T01:44:41.4614404Z else:
2021-04-13T01:44:41.4615224Z self.putheader('Content-Length', str(content_length))
2021-04-13T01:44:41.4615831Z else:
2021-04-13T01:44:41.4616270Z encode_chunked = False
2021-04-13T01:44:41.4616672Z
2021-04-13T01:44:41.4617128Z for hdr, value in headers.items():
2021-04-13T01:44:41.4617736Z self.putheader(hdr, value)
2021-04-13T01:44:41.4618275Z if isinstance(body, str):
2021-04-13T01:44:41.4618851Z # RFC 2616 Section 3.7.1 says that text default has a
2021-04-13T01:44:41.4619602Z # default charset of iso-8859-1.
2021-04-13T01:44:41.4620282Z body = _encode(body, 'body')
2021-04-13T01:44:41.4620945Z > self.endheaders(body, encode_chunked=encode_chunked)
2021-04-13T01:44:41.4621427Z
2021-04-13T01:44:41.4622282Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/http/client.py:1323:
2021-04-13T01:44:41.4623001Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4623390Z
2021-04-13T01:44:41.4624145Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4624982Z message_body = None
2021-04-13T01:44:41.4625259Z
2021-04-13T01:44:41.4625850Z def endheaders(self, message_body=None, *, encode_chunked=False):
2021-04-13T01:44:41.4626657Z """Indicate that the last header line has been sent to the server.
2021-04-13T01:44:41.4627214Z
2021-04-13T01:44:41.4627800Z This method sends the request to the server. The optional message_body
2021-04-13T01:44:41.4628653Z argument can be used to pass a message body associated with the
2021-04-13T01:44:41.4629246Z request.
2021-04-13T01:44:41.4629612Z """
2021-04-13T01:44:41.4630041Z if self.__state == _CS_REQ_STARTED:
2021-04-13T01:44:41.4630563Z self.__state = _CS_REQ_SENT
2021-04-13T01:44:41.4630983Z else:
2021-04-13T01:44:41.4635283Z raise CannotSendHeader()
2021-04-13T01:44:41.4636093Z > self._send_output(message_body, encode_chunked=encode_chunked)
2021-04-13T01:44:41.4639471Z
2021-04-13T01:44:41.4640585Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/http/client.py:1272:
2021-04-13T01:44:41.4644283Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4644633Z
2021-04-13T01:44:41.4645435Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4650089Z message_body = None, encode_chunked = False
2021-04-13T01:44:41.4650465Z
2021-04-13T01:44:41.4651105Z def _send_output(self, message_body=None, encode_chunked=False):
2021-04-13T01:44:41.4654931Z """Send the currently buffered request and clear the buffer.
2021-04-13T01:44:41.4655604Z
2021-04-13T01:44:41.4656093Z Appends an extra \\r\\n to the buffer.
2021-04-13T01:44:41.4659592Z A message_body may be specified, to be appended to the request.
2021-04-13T01:44:41.4660207Z """
2021-04-13T01:44:41.4664135Z self._buffer.extend((b"", b""))
2021-04-13T01:44:41.4664763Z msg = b"\r\n".join(self._buffer)
2021-04-13T01:44:41.4665278Z del self._buffer[:]
2021-04-13T01:44:41.4668229Z > self.send(msg)
2021-04-13T01:44:41.4668601Z
2021-04-13T01:44:41.4669659Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/http/client.py:1032:
2021-04-13T01:44:41.4672860Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4673152Z
2021-04-13T01:44:41.4674017Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4676151Z data = b'GET /status/ HTTP/1.1\r\nHost: fv-az212-648:8787\r\nUser-Agent: python-requests/2.25.1\r\nAccept-Encoding: gzip, deflate\r\nAccept: */*\r\nConnection: keep-alive\r\n\r\n'
2021-04-13T01:44:41.4679816Z
2021-04-13T01:44:41.4680303Z def send(self, data):
2021-04-13T01:44:41.4681007Z """Send `data' to the server.
2021-04-13T01:44:41.4684512Z ``data`` can be a string object, a bytes object, an array object, a
2021-04-13T01:44:41.4685678Z file-like object that supports a .read() method, or an iterable object.
2021-04-13T01:44:41.4688781Z """
2021-04-13T01:44:41.4689181Z
2021-04-13T01:44:41.4689682Z if self.sock is None:
2021-04-13T01:44:41.4694774Z if self.auto_open:
2021-04-13T01:44:41.4695390Z > self.connect()
2021-04-13T01:44:41.4695687Z
2021-04-13T01:44:41.4699419Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/http/client.py:972:
2021-04-13T01:44:41.4700221Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4700558Z
2021-04-13T01:44:41.4704547Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4705281Z
2021-04-13T01:44:41.4705753Z def connect(self):
2021-04-13T01:44:41.4706230Z > conn = self._new_conn()
2021-04-13T01:44:41.4715016Z
2021-04-13T01:44:41.4716286Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connection.py:200:
2021-04-13T01:44:41.4717411Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4721287Z
2021-04-13T01:44:41.4722086Z self = <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>
2021-04-13T01:44:41.4722794Z
2021-04-13T01:44:41.4723291Z def _new_conn(self):
2021-04-13T01:44:41.4726788Z """Establish a socket connection and set nodelay settings on it.
2021-04-13T01:44:41.4727387Z
2021-04-13T01:44:41.4730385Z :return: New socket connection.
2021-04-13T01:44:41.4730914Z """
2021-04-13T01:44:41.4731376Z extra_kw = {}
2021-04-13T01:44:41.4734450Z if self.source_address:
2021-04-13T01:44:41.4735195Z extra_kw["source_address"] = self.source_address
2021-04-13T01:44:41.4735742Z
2021-04-13T01:44:41.4738647Z if self.socket_options:
2021-04-13T01:44:41.4739308Z extra_kw["socket_options"] = self.socket_options
2021-04-13T01:44:41.4742348Z
2021-04-13T01:44:41.4742760Z try:
2021-04-13T01:44:41.4743337Z conn = connection.create_connection(
2021-04-13T01:44:41.4746689Z (self._dns_host, self.port), self.timeout, **extra_kw
2021-04-13T01:44:41.4747306Z )
2021-04-13T01:44:41.4747710Z
2021-04-13T01:44:41.4750613Z except SocketTimeout:
2021-04-13T01:44:41.4751311Z raise ConnectTimeoutError(
2021-04-13T01:44:41.4751861Z self,
2021-04-13T01:44:41.4755930Z "Connection to %s timed out. (connect timeout=%s)"
2021-04-13T01:44:41.4756653Z % (self.host, self.timeout),
2021-04-13T01:44:41.4760032Z )
2021-04-13T01:44:41.4760406Z
2021-04-13T01:44:41.4760892Z except SocketError as e:
2021-04-13T01:44:41.4764811Z raise NewConnectionError(
2021-04-13T01:44:41.4765544Z > self, "Failed to establish a new connection: %s" % e
2021-04-13T01:44:41.4766182Z )
2021-04-13T01:44:41.4770442Z E urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f6e9a188310>: Failed to establish a new connection: [Errno 110] Connection timed out
2021-04-13T01:44:41.4771860Z
2021-04-13T01:44:41.4773533Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connection.py:182: NewConnectionError
2021-04-13T01:44:41.4777124Z
2021-04-13T01:44:41.4777783Z During handling of the above exception, another exception occurred:
2021-04-13T01:44:41.4778351Z
2021-04-13T01:44:41.4779111Z self = <requests.adapters.HTTPAdapter object at 0x7f6e9b0f8690>
2021-04-13T01:44:41.4782502Z request = <PreparedRequest [GET]>, stream = False
2021-04-13T01:44:41.4783478Z timeout = Timeout(connect=None, read=None, total=None), verify = True
2021-04-13T01:44:41.4784288Z cert = None, proxies = OrderedDict()
2021-04-13T01:44:41.4784648Z
2021-04-13T01:44:41.4787912Z def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
2021-04-13T01:44:41.4791417Z """Sends PreparedRequest object. Returns Response object.
2021-04-13T01:44:41.4792377Z
2021-04-13T01:44:41.4793184Z :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
2021-04-13T01:44:41.4796875Z :param stream: (optional) Whether to stream the request content.
2021-04-13T01:44:41.4797810Z :param timeout: (optional) How long to wait for the server to send
2021-04-13T01:44:41.4798620Z data before giving up, as a float, or a :ref:`(connect timeout,
2021-04-13T01:44:41.4815902Z read timeout) <timeouts>` tuple.
2021-04-13T01:44:41.4816627Z :type timeout: float or tuple or urllib3 Timeout object
2021-04-13T01:44:41.4821579Z :param verify: (optional) Either a boolean, in which case it controls whether
2021-04-13T01:44:41.4822900Z we verify the server's TLS certificate, or a string, in which case it
2021-04-13T01:44:41.4823710Z must be a path to a CA bundle to use
2021-04-13T01:44:41.4827829Z :param cert: (optional) Any user-provided SSL certificate to be trusted.
2021-04-13T01:44:41.4828917Z :param proxies: (optional) The proxies dictionary to apply to the request.
2021-04-13T01:44:41.4829725Z :rtype: requests.Response
2021-04-13T01:44:41.4832784Z """
2021-04-13T01:44:41.4833217Z
2021-04-13T01:44:41.4833576Z try:
2021-04-13T01:44:41.4836704Z conn = self.get_connection(request.url, proxies)
2021-04-13T01:44:41.4837471Z except LocationValueError as e:
2021-04-13T01:44:41.4841428Z raise InvalidURL(e, request=request)
2021-04-13T01:44:41.4841962Z
2021-04-13T01:44:41.4842606Z self.cert_verify(conn, request.url, verify, cert)
2021-04-13T01:44:41.4845764Z url = self.request_url(request, proxies)
2021-04-13T01:44:41.4846843Z self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
2021-04-13T01:44:41.4847623Z
2021-04-13T01:44:41.4851111Z chunked = not (request.body is None or 'Content-Length' in request.headers)
2021-04-13T01:44:41.4851840Z
2021-04-13T01:44:41.4852328Z if isinstance(timeout, tuple):
2021-04-13T01:44:41.4855881Z try:
2021-04-13T01:44:41.4856534Z connect, read = timeout
2021-04-13T01:44:41.4859903Z timeout = TimeoutSauce(connect=connect, read=read)
2021-04-13T01:44:41.4862638Z except ValueError as e:
2021-04-13T01:44:41.4863344Z # this may raise a string formatting error.
2021-04-13T01:44:41.4867311Z err = ("Invalid timeout {}. Pass a (connect, read) "
2021-04-13T01:44:41.4868466Z "timeout tuple, or a single float to set "
2021-04-13T01:44:41.4869223Z "both timeouts to the same value".format(timeout))
2021-04-13T01:44:41.4872579Z raise ValueError(err)
2021-04-13T01:44:41.4873245Z elif isinstance(timeout, TimeoutSauce):
2021-04-13T01:44:41.4876468Z pass
2021-04-13T01:44:41.4876921Z else:
2021-04-13T01:44:41.4877605Z timeout = TimeoutSauce(connect=timeout, read=timeout)
2021-04-13T01:44:41.4880747Z
2021-04-13T01:44:41.4881152Z try:
2021-04-13T01:44:41.4881639Z if not chunked:
2021-04-13T01:44:41.4884850Z resp = conn.urlopen(
2021-04-13T01:44:41.4885520Z method=request.method,
2021-04-13T01:44:41.4886045Z url=url,
2021-04-13T01:44:41.4890113Z body=request.body,
2021-04-13T01:44:41.4890723Z headers=request.headers,
2021-04-13T01:44:41.4893835Z redirect=False,
2021-04-13T01:44:41.4894590Z assert_same_host=False,
2021-04-13T01:44:41.4895220Z preload_content=False,
2021-04-13T01:44:41.4898393Z decode_content=False,
2021-04-13T01:44:41.4899046Z retries=self.max_retries,
2021-04-13T01:44:41.4899594Z > timeout=timeout
2021-04-13T01:44:41.4902477Z )
2021-04-13T01:44:41.4902779Z
2021-04-13T01:44:41.4904047Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/requests/adapters.py:449:
2021-04-13T01:44:41.4907520Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.4907874Z
2021-04-13T01:44:41.4908857Z self = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f6e99e3a090>
2021-04-13T01:44:41.4912588Z method = 'GET', url = '/status/', body = None
2021-04-13T01:44:41.4913898Z headers = {'User-Agent': 'python-requests/2.25.1', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}
2021-04-13T01:44:41.4915151Z retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
2021-04-13T01:44:41.4918683Z redirect = False, assert_same_host = False
2021-04-13T01:44:41.4919517Z timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
2021-04-13T01:44:41.4920329Z release_conn = False, chunked = False, body_pos = None
2021-04-13T01:44:41.4924379Z response_kw = {'decode_content': False, 'preload_content': False}
2021-04-13T01:44:41.4925578Z parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/status/', query=None, fragment=None)
2021-04-13T01:44:41.4928936Z destination_scheme = None, conn = None, release_this_conn = True
2021-04-13T01:44:41.4929748Z http_tunnel_required = False, err = None, clean_exit = False
2021-04-13T01:44:41.4930239Z
2021-04-13T01:44:41.4933357Z def urlopen(
2021-04-13T01:44:41.4933827Z self,
2021-04-13T01:44:41.4934229Z method,
2021-04-13T01:44:41.4937359Z url,
2021-04-13T01:44:41.4937757Z body=None,
2021-04-13T01:44:41.4938254Z headers=None,
2021-04-13T01:44:41.4941064Z retries=None,
2021-04-13T01:44:41.4941580Z redirect=True,
2021-04-13T01:44:41.4942068Z assert_same_host=True,
2021-04-13T01:44:41.4945000Z timeout=_Default,
2021-04-13T01:44:41.4945496Z pool_timeout=None,
2021-04-13T01:44:41.4948551Z release_conn=None,
2021-04-13T01:44:41.4949032Z chunked=False,
2021-04-13T01:44:41.4949483Z body_pos=None,
2021-04-13T01:44:41.4952348Z **response_kw
2021-04-13T01:44:41.4952797Z ):
2021-04-13T01:44:41.4953195Z """
2021-04-13T01:44:41.4956171Z Get a connection from the pool and perform an HTTP request. This is the
2021-04-13T01:44:41.4957439Z lowest level call for making a request, so you'll need to specify all
2021-04-13T01:44:41.4958106Z the raw details.
2021-04-13T01:44:41.4961475Z
2021-04-13T01:44:41.4961900Z .. note::
2021-04-13T01:44:41.4962244Z
2021-04-13T01:44:41.4963148Z More commonly, it's appropriate to use a convenience method provided
2021-04-13T01:44:41.4963977Z by :class:`.RequestMethods`, such as :meth:`request`.
2021-04-13T01:44:41.4964514Z
2021-04-13T01:44:41.4964844Z .. note::
2021-04-13T01:44:41.4965185Z
2021-04-13T01:44:41.4965681Z `release_conn` will only behave as expected if
2021-04-13T01:44:41.4966375Z `preload_content=False` because we want to make
2021-04-13T01:44:41.4967140Z `preload_content=False` the default behaviour someday soon without
2021-04-13T01:44:41.4968277Z breaking backwards compatibility.
2021-04-13T01:44:41.4968901Z
2021-04-13T01:44:41.4969294Z :param method:
2021-04-13T01:44:41.4969853Z HTTP request method (such as GET, POST, PUT, etc.)
2021-04-13T01:44:41.4970361Z
2021-04-13T01:44:41.4970701Z :param url:
2021-04-13T01:44:41.4971180Z The URL to perform the request on.
2021-04-13T01:44:41.4971638Z
2021-04-13T01:44:41.4972136Z :param body:
2021-04-13T01:44:41.4973072Z Data to send in the request body, either :class:`str`, :class:`bytes`,
2021-04-13T01:44:41.4974141Z an iterable of :class:`str`/:class:`bytes`, or a file-like object.
2021-04-13T01:44:41.4974695Z
2021-04-13T01:44:41.4975069Z :param headers:
2021-04-13T01:44:41.4975867Z Dictionary of custom headers to send, such as User-Agent,
2021-04-13T01:44:41.4976886Z If-None-Match, etc. If None, pool headers are used. If provided,
2021-04-13T01:44:41.4978003Z these headers completely replace any pool-specific headers.
2021-04-13T01:44:41.4979146Z
2021-04-13T01:44:41.4979868Z :param retries:
2021-04-13T01:44:41.4980483Z Configure the number of retries to allow before raising a
2021-04-13T01:44:41.4981442Z :class:`~urllib3.exceptions.MaxRetryError` exception.
2021-04-13T01:44:41.4982160Z
2021-04-13T01:44:41.4982702Z Pass ``None`` to retry until you receive a response. Pass a
2021-04-13T01:44:41.4983860Z :class:`~urllib3.util.retry.Retry` object for fine-grained control
2021-04-13T01:44:41.4984671Z over different types of retries.
2021-04-13T01:44:41.4985399Z Pass an integer number to retry connection errors that many times,
2021-04-13T01:44:41.4986305Z but no other types of errors. Pass zero to never retry.
2021-04-13T01:44:41.4986823Z
2021-04-13T01:44:41.4987390Z If ``False``, then retries are disabled and any exception is raised
2021-04-13T01:44:41.4988270Z immediately. Also, instead of raising a MaxRetryError on redirects,
2021-04-13T01:44:41.4989064Z the redirect response will be returned.
2021-04-13T01:44:41.4989561Z
2021-04-13T01:44:41.4990208Z :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
2021-04-13T01:44:41.4990868Z
2021-04-13T01:44:41.4991246Z :param redirect:
2021-04-13T01:44:41.4991890Z If True, automatically handle redirects (status codes 301, 302,
2021-04-13T01:44:41.4992671Z 303, 307, 308). Each redirect counts as a retry. Disabling retries
2021-04-13T01:44:41.4993318Z will disable redirect, too.
2021-04-13T01:44:41.4993735Z
2021-04-13T01:44:41.4994145Z :param assert_same_host:
2021-04-13T01:44:41.4994755Z If ``True``, will make sure that the host of the pool requests is
2021-04-13T01:44:41.4995593Z consistent else will raise HostChangedError. When ``False``, you can
2021-04-13T01:44:41.4996438Z use the pool on an HTTP proxy and request foreign hosts.
2021-04-13T01:44:41.4996952Z
2021-04-13T01:44:41.4997330Z :param timeout:
2021-04-13T01:44:41.4997934Z If specified, overrides the default timeout for this one
2021-04-13T01:44:41.4998705Z request. It may be a float (in seconds) or an instance of
2021-04-13T01:44:41.4999387Z :class:`urllib3.util.Timeout`.
2021-04-13T01:44:41.4999880Z
2021-04-13T01:44:41.5000265Z :param pool_timeout:
2021-04-13T01:44:41.5000880Z If set and the pool is set to block=True, then this method will
2021-04-13T01:44:41.5001669Z block for ``pool_timeout`` seconds and raise EmptyPoolError if no
2021-04-13T01:44:41.5002470Z connection is available within the time period.
2021-04-13T01:44:41.5002974Z
2021-04-13T01:44:41.5003371Z :param release_conn:
2021-04-13T01:44:41.5004008Z If False, then the urlopen call will not release the connection
2021-04-13T01:44:41.5004827Z back into the pool once a response is received (but will release if
2021-04-13T01:44:41.5005619Z you read the entire contents of the response such as when
2021-04-13T01:44:41.5006591Z `preload_content=True`). This is useful if you're not preloading
2021-04-13T01:44:41.5007557Z the response's content immediately. You will need to call
2021-04-13T01:44:41.5008430Z ``r.release_conn()`` on the response ``r`` to return the connection
2021-04-13T01:44:41.5009166Z back into the pool. If None, it takes the value of
2021-04-13T01:44:41.5009995Z ``response_kw.get('preload_content', True)``.
2021-04-13T01:44:41.5010489Z
2021-04-13T01:44:41.5010861Z :param chunked:
2021-04-13T01:44:41.5011456Z If True, urllib3 will send the body using chunked transfer
2021-04-13T01:44:41.5012248Z encoding. Otherwise, urllib3 will send the body using the standard
2021-04-13T01:44:41.5013489Z content-length form. Defaults to False.
2021-04-13T01:44:41.5014020Z
2021-04-13T01:44:41.5014409Z :param int body_pos:
2021-04-13T01:44:41.5015217Z Position to seek to in file-like body in the event of a retry or
2021-04-13T01:44:41.5018004Z redirect. Typically this won't need to be set because urllib3 will
2021-04-13T01:44:41.5018984Z auto-populate the value when needed.
2021-04-13T01:44:41.5019488Z
2021-04-13T01:44:41.5019881Z :param \\**response_kw:
2021-04-13T01:44:41.5020467Z Additional parameters are passed to
2021-04-13T01:44:41.5021579Z :meth:`urllib3.response.HTTPResponse.from_httplib`
2021-04-13T01:44:41.5022325Z """
2021-04-13T01:44:41.5022804Z
2021-04-13T01:44:41.5023207Z parsed_url = parse_url(url)
2021-04-13T01:44:41.5023797Z destination_scheme = parsed_url.scheme
2021-04-13T01:44:41.5024278Z
2021-04-13T01:44:41.5024663Z if headers is None:
2021-04-13T01:44:41.5025148Z headers = self.headers
2021-04-13T01:44:41.5025571Z
2021-04-13T01:44:41.5026018Z if not isinstance(retries, Retry):
2021-04-13T01:44:41.5026825Z retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
2021-04-13T01:44:41.5027472Z
2021-04-13T01:44:41.5027865Z if release_conn is None:
2021-04-13T01:44:41.5028507Z release_conn = response_kw.get("preload_content", True)
2021-04-13T01:44:41.5029033Z
2021-04-13T01:44:41.5029385Z # Check host
2021-04-13T01:44:41.5029898Z if assert_same_host and not self.is_same_host(url):
2021-04-13T01:44:41.5030599Z raise HostChangedError(self, url, retries)
2021-04-13T01:44:41.5031122Z
2021-04-13T01:44:41.5031891Z # Ensure that the URL we're connecting to is properly encoded
2021-04-13T01:44:41.5032534Z if url.startswith("/"):
2021-04-13T01:44:41.5033110Z url = six.ensure_str(_encode_target(url))
2021-04-13T01:44:41.5033593Z else:
2021-04-13T01:44:41.5034083Z url = six.ensure_str(parsed_url.url)
2021-04-13T01:44:41.5034536Z
2021-04-13T01:44:41.5034871Z conn = None
2021-04-13T01:44:41.5035220Z
2021-04-13T01:44:41.5035725Z # Track whether `conn` needs to be released before
2021-04-13T01:44:41.5036537Z # returning/raising/recursing. Update this variable if necessary, and
2021-04-13T01:44:41.5037428Z # leave `release_conn` constant throughout the function. That way, if
2021-04-13T01:44:41.5038296Z # the function recurses, the original value of `release_conn` will be
2021-04-13T01:44:41.5039142Z # passed down into the recursive call, and its value will be respected.
2021-04-13T01:44:41.5039735Z #
2021-04-13T01:44:41.5040144Z # See issue #651 [1] for details.
2021-04-13T01:44:41.5040563Z #
2021-04-13T01:44:41.5041128Z # [1] <https://github.com/urllib3/urllib3/issues/651>
2021-04-13T01:44:41.5041813Z release_this_conn = release_conn
2021-04-13T01:44:41.5042244Z
2021-04-13T01:44:41.5042803Z http_tunnel_required = connection_requires_http_tunnel(
2021-04-13T01:44:41.5043586Z self.proxy, self.proxy_config, destination_scheme
2021-04-13T01:44:41.5044147Z )
2021-04-13T01:44:41.5044450Z
2021-04-13T01:44:41.5045028Z # Merge the proxy headers. Only done when not using HTTP CONNECT. We
2021-04-13T01:44:41.5045988Z # have to copy the headers dict so we can safely change it without those
2021-04-13T01:44:41.5046934Z # changes being reflected in anyone else's copy.
2021-04-13T01:44:41.5047557Z if not http_tunnel_required:
2021-04-13T01:44:41.5048091Z headers = headers.copy()
2021-04-13T01:44:41.5048728Z headers.update(self.proxy_headers)
2021-04-13T01:44:41.5049221Z
2021-04-13T01:44:41.5049831Z # Must keep the exception bound to a separate variable or else Python 3
2021-04-13T01:44:41.5050613Z # complains about UnboundLocalError.
2021-04-13T01:44:41.5051157Z err = None
2021-04-13T01:44:41.5051495Z
2021-04-13T01:44:41.5052068Z # Keep track of whether we cleanly exited the except block. This
2021-04-13T01:44:41.5053004Z # ensures we do proper cleanup in finally.
2021-04-13T01:44:41.5053556Z clean_exit = False
2021-04-13T01:44:41.5053922Z
2021-04-13T01:44:41.5054475Z # Rewind body position, if needed. Record current position
2021-04-13T01:44:41.5055233Z # for future rewinds in the event of a redirect/retry.
2021-04-13T01:44:41.5055894Z body_pos = set_file_position(body, body_pos)
2021-04-13T01:44:41.5056366Z
2021-04-13T01:44:41.5056682Z try:
2021-04-13T01:44:41.5057174Z # Request a connection from the queue.
2021-04-13T01:44:41.5057887Z timeout_obj = self._get_timeout(timeout)
2021-04-13T01:44:41.5058523Z conn = self._get_conn(timeout=pool_timeout)
2021-04-13T01:44:41.5058983Z
2021-04-13T01:44:41.5059501Z conn.timeout = timeout_obj.connect_timeout
2021-04-13T01:44:41.5059994Z
2021-04-13T01:44:41.5060525Z is_new_proxy_conn = self.proxy is not None and not getattr(
2021-04-13T01:44:41.5061099Z conn, "sock", None
2021-04-13T01:44:41.5061468Z )
2021-04-13T01:44:41.5061971Z if is_new_proxy_conn and http_tunnel_required:
2021-04-13T01:44:41.5062550Z self._prepare_proxy(conn)
2021-04-13T01:44:41.5062984Z
2021-04-13T01:44:41.5063505Z # Make the request on the httplib connection object.
2021-04-13T01:44:41.5064190Z httplib_response = self._make_request(
2021-04-13T01:44:41.5064679Z conn,
2021-04-13T01:44:41.5065062Z method,
2021-04-13T01:44:41.5065424Z url,
2021-04-13T01:44:41.5065850Z timeout=timeout_obj,
2021-04-13T01:44:41.5066280Z body=body,
2021-04-13T01:44:41.5066710Z headers=headers,
2021-04-13T01:44:41.5067188Z chunked=chunked,
2021-04-13T01:44:41.5067572Z )
2021-04-13T01:44:41.5067900Z
2021-04-13T01:44:41.5068658Z # If we're going to release the connection in ``finally:``, then
2021-04-13T01:44:41.5069677Z # the response doesn't need to know about the connection. Otherwise
2021-04-13T01:44:41.5070676Z # it will also try to release it and we'll have a double-release
2021-04-13T01:44:41.5071264Z # mess.
2021-04-13T01:44:41.5071796Z response_conn = conn if not release_conn else None
2021-04-13T01:44:41.5072302Z
2021-04-13T01:44:41.5072796Z # Pass method to Response for length checking
2021-04-13T01:44:41.5073443Z response_kw["request_method"] = method
2021-04-13T01:44:41.5073901Z
2021-04-13T01:44:41.5074610Z # Import httplib's response into our own wrapper object
2021-04-13T01:44:41.5075388Z response = self.ResponseCls.from_httplib(
2021-04-13T01:44:41.5076041Z httplib_response,
2021-04-13T01:44:41.5076497Z pool=self,
2021-04-13T01:44:41.5076973Z connection=response_conn,
2021-04-13T01:44:41.5077503Z retries=retries,
2021-04-13T01:44:41.5077948Z **response_kw
2021-04-13T01:44:41.5078337Z )
2021-04-13T01:44:41.5078651Z
2021-04-13T01:44:41.5079312Z # Everything went great!
2021-04-13T01:44:41.5079931Z clean_exit = True
2021-04-13T01:44:41.5080482Z
2021-04-13T01:44:41.5080933Z except EmptyPoolError:
2021-04-13T01:44:41.5081829Z # Didn't get a connection from the pool, no need to clean up
2021-04-13T01:44:41.5082429Z clean_exit = True
2021-04-13T01:44:41.5082897Z release_this_conn = False
2021-04-13T01:44:41.5083337Z raise
2021-04-13T01:44:41.5083664Z
2021-04-13T01:44:41.5084018Z except (
2021-04-13T01:44:41.5084438Z TimeoutError,
2021-04-13T01:44:41.5084925Z HTTPException,
2021-04-13T01:44:41.5085397Z SocketError,
2021-04-13T01:44:41.5085870Z ProtocolError,
2021-04-13T01:44:41.5086345Z BaseSSLError,
2021-04-13T01:44:41.5086771Z SSLError,
2021-04-13T01:44:41.5087258Z CertificateError,
2021-04-13T01:44:41.5087703Z ) as e:
2021-04-13T01:44:41.5088296Z # Discard the connection for these exceptions. It will be
2021-04-13T01:44:41.5089014Z # replaced during the next _get_conn() call.
2021-04-13T01:44:41.5089556Z clean_exit = False
2021-04-13T01:44:41.5090207Z if isinstance(e, (BaseSSLError, CertificateError)):
2021-04-13T01:44:41.5090867Z e = SSLError(e)
2021-04-13T01:44:41.5091726Z elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
2021-04-13T01:44:41.5092789Z e = ProxyError("Cannot connect to proxy.", e)
2021-04-13T01:44:41.5093543Z elif isinstance(e, (SocketError, HTTPException)):
2021-04-13T01:44:41.5094333Z e = ProtocolError("Connection aborted.", e)
2021-04-13T01:44:41.5094849Z
2021-04-13T01:44:41.5095336Z retries = retries.increment(
2021-04-13T01:44:41.5096377Z > method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
2021-04-13T01:44:41.5096966Z )
2021-04-13T01:44:41.5097194Z
2021-04-13T01:44:41.5099734Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/connectionpool.py:756:
2021-04-13T01:44:41.5100653Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.5100945Z
2021-04-13T01:44:41.5101542Z self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
2021-04-13T01:44:41.5102523Z method = 'GET', url = '/status/', response = None
2021-04-13T01:44:41.5104150Z error = NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f6e9a188310>: Failed to establish a new connection: [Errno 110] Connection timed out')
2021-04-13T01:44:41.5105952Z _pool = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f6e99e3a090>
2021-04-13T01:44:41.5107082Z _stacktrace = <traceback object at 0x7f6e9a1268c0>
2021-04-13T01:44:41.5107481Z
2021-04-13T01:44:41.5107855Z def increment(
2021-04-13T01:44:41.5108245Z self,
2021-04-13T01:44:41.5108629Z method=None,
2021-04-13T01:44:41.5109012Z url=None,
2021-04-13T01:44:41.5109423Z response=None,
2021-04-13T01:44:41.5109842Z error=None,
2021-04-13T01:44:41.5110232Z _pool=None,
2021-04-13T01:44:41.5110667Z _stacktrace=None,
2021-04-13T01:44:41.5111046Z ):
2021-04-13T01:44:41.5111615Z """Return a new Retry object with incremented retry counters.
2021-04-13T01:44:41.5112166Z
2021-04-13T01:44:41.5112746Z :param response: A response object, or None, if the server did not
2021-04-13T01:44:41.5113397Z return a response.
2021-04-13T01:44:41.5114172Z :type response: :class:`~urllib3.response.HTTPResponse`
2021-04-13T01:44:41.5115189Z :param Exception error: An error encountered during the request, or
2021-04-13T01:44:41.5116021Z None if the response was received successfully.
2021-04-13T01:44:41.5116538Z
2021-04-13T01:44:41.5116959Z :return: A new ``Retry`` object.
2021-04-13T01:44:41.5117367Z """
2021-04-13T01:44:41.5117802Z if self.total is False and error:
2021-04-13T01:44:41.5118765Z # Disabled, indicate to re-raise the error.
2021-04-13T01:44:41.5119483Z raise six.reraise(type(error), error, _stacktrace)
2021-04-13T01:44:41.5120021Z
2021-04-13T01:44:41.5120393Z total = self.total
2021-04-13T01:44:41.5120849Z if total is not None:
2021-04-13T01:44:41.5121397Z total -= 1
2021-04-13T01:44:41.5121748Z
2021-04-13T01:44:41.5122152Z connect = self.connect
2021-04-13T01:44:41.5122629Z read = self.read
2021-04-13T01:44:41.5123113Z redirect = self.redirect
2021-04-13T01:44:41.5123639Z status_count = self.status
2021-04-13T01:44:41.5124136Z other = self.other
2021-04-13T01:44:41.5124569Z cause = "unknown"
2021-04-13T01:44:41.5124996Z status = None
2021-04-13T01:44:41.5125440Z redirect_location = None
2021-04-13T01:44:41.5125855Z
2021-04-13T01:44:41.5126336Z if error and self._is_connection_error(error):
2021-04-13T01:44:41.5126894Z # Connect retry?
2021-04-13T01:44:41.5127344Z if connect is False:
2021-04-13T01:44:41.5127961Z raise six.reraise(type(error), error, _stacktrace)
2021-04-13T01:44:41.5128580Z elif connect is not None:
2021-04-13T01:44:41.5129182Z connect -= 1
2021-04-13T01:44:41.5129627Z
2021-04-13T01:44:41.5130334Z elif error and self._is_read_error(error):
2021-04-13T01:44:41.5130855Z # Read retry?
2021-04-13T01:44:41.5131680Z if read is False or not self._is_method_retryable(method):
2021-04-13T01:44:41.5132437Z raise six.reraise(type(error), error, _stacktrace)
2021-04-13T01:44:41.5133143Z elif read is not None:
2021-04-13T01:44:41.5133767Z read -= 1
2021-04-13T01:44:41.5134108Z
2021-04-13T01:44:41.5134474Z elif error:
2021-04-13T01:44:41.5134874Z # Other retry?
2021-04-13T01:44:41.5135318Z if other is not None:
2021-04-13T01:44:41.5135860Z other -= 1
2021-04-13T01:44:41.5136211Z
2021-04-13T01:44:41.5136776Z elif response and response.get_redirect_location():
2021-04-13T01:44:41.5137396Z # Redirect retry?
2021-04-13T01:44:41.5137878Z if redirect is not None:
2021-04-13T01:44:41.5138470Z redirect -= 1
2021-04-13T01:44:41.5138968Z cause = "too many redirects"
2021-04-13T01:44:41.5139633Z redirect_location = response.get_redirect_location()
2021-04-13T01:44:41.5140319Z status = response.status
2021-04-13T01:44:41.5140753Z
2021-04-13T01:44:41.5141092Z else:
2021-04-13T01:44:41.5141631Z # Incrementing because of a server error like a 500 in
2021-04-13T01:44:41.5142411Z # status_forcelist and the given method is in the allowed_methods
2021-04-13T01:44:41.5143195Z cause = ResponseError.GENERIC_ERROR
2021-04-13T01:44:41.5143914Z if response and response.status:
2021-04-13T01:44:41.5144488Z if status_count is not None:
2021-04-13T01:44:41.5145109Z status_count -= 1
2021-04-13T01:44:41.5145933Z cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
2021-04-13T01:44:41.5146807Z status = response.status
2021-04-13T01:44:41.5147256Z
2021-04-13T01:44:41.5149363Z history = self.history + (
2021-04-13T01:44:41.5150103Z RequestHistory(method, url, error, status, redirect_location),
2021-04-13T01:44:41.5150701Z )
2021-04-13T01:44:41.5151025Z
2021-04-13T01:44:41.5151412Z new_retry = self.new(
2021-04-13T01:44:41.5151855Z total=total,
2021-04-13T01:44:41.5152280Z connect=connect,
2021-04-13T01:44:41.5152711Z read=read,
2021-04-13T01:44:41.5153150Z redirect=redirect,
2021-04-13T01:44:41.5153629Z status=status_count,
2021-04-13T01:44:41.5154072Z other=other,
2021-04-13T01:44:41.5154491Z history=history,
2021-04-13T01:44:41.5155018Z )
2021-04-13T01:44:41.5155330Z
2021-04-13T01:44:41.5155763Z if new_retry.is_exhausted():
2021-04-13T01:44:41.5156487Z > raise MaxRetryError(_pool, url, error or ResponseError(cause))
2021-04-13T01:44:41.5159209Z E urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='fv-az212-648', port=8787): Max retries exceeded with url: /status/ (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f6e9a188310>: Failed to establish a new connection: [Errno 110] Connection timed out'))
2021-04-13T01:44:41.5161015Z
2021-04-13T01:44:41.5162083Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/urllib3/util/retry.py:574: MaxRetryError
2021-04-13T01:44:41.5162832Z
2021-04-13T01:44:41.5163438Z During handling of the above exception, another exception occurred:
2021-04-13T01:44:41.5163955Z
2021-04-13T01:44:41.5164791Z loop = <tornado.platform.asyncio.AsyncIOLoop object at 0x7f6e99dd7e90>
2021-04-13T01:44:41.5165538Z
2021-04-13T01:44:41.5165938Z def test_dashboard(loop):
2021-04-13T01:44:41.5166513Z pytest.importorskip("bokeh")
2021-04-13T01:44:41.5167007Z
2021-04-13T01:44:41.5168214Z with popen(["dask-scheduler"]) as proc:
2021-04-13T01:44:41.5168821Z for line in proc.stderr:
2021-04-13T01:44:41.5169458Z if b"dashboard at" in line:
2021-04-13T01:44:41.5170340Z dashboard_port = int(line.decode().split(":")[-1].strip())
2021-04-13T01:44:41.5170902Z break
2021-04-13T01:44:41.5171280Z else:
2021-04-13T01:44:41.5172432Z raise Exception("dashboard not found")
2021-04-13T01:44:41.5173123Z
2021-04-13T01:44:41.5173698Z with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
2021-04-13T01:44:41.5174313Z pass
2021-04-13T01:44:41.5174643Z
2021-04-13T01:44:41.5175063Z names = ["localhost", "127.0.0.1", get_ip()]
2021-04-13T01:44:41.5175625Z if "linux" in sys.platform:
2021-04-13T01:44:41.5176256Z names.append(socket.gethostname())
2021-04-13T01:44:41.5176790Z
2021-04-13T01:44:41.5177144Z start = time()
2021-04-13T01:44:41.5177555Z while True:
2021-04-13T01:44:41.5177921Z try:
2021-04-13T01:44:41.5178397Z # All addresses should respond
2021-04-13T01:44:41.5178901Z for name in names:
2021-04-13T01:44:41.5179509Z uri = "http://%s:%d/status/" % (name, dashboard_port)
2021-04-13T01:44:41.5180166Z > response = requests.get(uri)
2021-04-13T01:44:41.5180540Z
2021-04-13T01:44:41.5181053Z distributed/cli/tests/test_dask_scheduler.py:89:
2021-04-13T01:44:41.5191715Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.5196037Z
2021-04-13T01:44:41.5198853Z url = 'http://fv-az212-648:8787/status/', params = None
2021-04-13T01:44:41.5201983Z kwargs = {'allow_redirects': True}
2021-04-13T01:44:41.5202415Z
2021-04-13T01:44:41.5205333Z def get(url, params=None, **kwargs):
2021-04-13T01:44:41.5205864Z r"""Sends a GET request.
2021-04-13T01:44:41.5260428Z
2021-04-13T01:44:41.5260948Z :param url: URL for the new :class:`Request` object.
2021-04-13T01:44:41.5261742Z :param params: (optional) Dictionary, list of tuples or bytes to send
2021-04-13T01:44:41.5262484Z in the query string for the :class:`Request`.
2021-04-13T01:44:41.5263199Z :param \*\*kwargs: Optional arguments that ``request`` takes.
2021-04-13T01:44:41.5264223Z :return: :class:`Response <Response>` object
2021-04-13T01:44:41.5264879Z :rtype: requests.Response
2021-04-13T01:44:41.5265337Z """
2021-04-13T01:44:41.5265664Z
2021-04-13T01:44:41.5266563Z kwargs.setdefault('allow_redirects', True)
2021-04-13T01:44:41.5267488Z > return request('get', url, params=params, **kwargs)
2021-04-13T01:44:41.5267893Z
2021-04-13T01:44:41.5269304Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/requests/api.py:76:
2021-04-13T01:44:41.5270126Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.5270421Z
2021-04-13T01:44:41.5271229Z method = 'get', url = 'http://fv-az212-648:8787/status/'
2021-04-13T01:44:41.5272142Z kwargs = {'allow_redirects': True, 'params': None}
2021-04-13T01:44:41.5272997Z session = <requests.sessions.Session object at 0x7f6e99ed3350>
2021-04-13T01:44:41.5273555Z
2021-04-13T01:44:41.5274217Z def request(method, url, **kwargs):
2021-04-13T01:44:41.5274866Z """Constructs and sends a :class:`Request <Request>`.
2021-04-13T01:44:41.5275379Z
2021-04-13T01:44:41.5276099Z :param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``.
2021-04-13T01:44:41.5277005Z :param url: URL for the new :class:`Request` object.
2021-04-13T01:44:41.5277773Z :param params: (optional) Dictionary, list of tuples or bytes to send
2021-04-13T01:44:41.5278517Z in the query string for the :class:`Request`.
2021-04-13T01:44:41.5279505Z :param data: (optional) Dictionary, list of tuples, bytes, or file-like
2021-04-13T01:44:41.5280405Z object to send in the body of the :class:`Request`.
2021-04-13T01:44:41.5281260Z :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
2021-04-13T01:44:41.5282289Z :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
2021-04-13T01:44:41.5283251Z :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
2021-04-13T01:44:41.5284883Z :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
2021-04-13T01:44:41.5286344Z ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
2021-04-13T01:44:41.5287690Z or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string
2021-04-13T01:44:41.5289104Z defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
2021-04-13T01:44:41.5289964Z to add for the file.
2021-04-13T01:44:41.5290634Z :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
2021-04-13T01:44:41.5291519Z :param timeout: (optional) How many seconds to wait for the server to send data
2021-04-13T01:44:41.5292325Z before giving up, as a float, or a :ref:`(connect timeout, read
2021-04-13T01:44:41.5293143Z timeout) <timeouts>` tuple.
2021-04-13T01:44:41.5293674Z :type timeout: float or tuple
2021-04-13T01:44:41.5294825Z :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
2021-04-13T01:44:41.5295778Z :type allow_redirects: bool
2021-04-13T01:44:41.5296537Z :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
2021-04-13T01:44:41.5297492Z :param verify: (optional) Either a boolean, in which case it controls whether we verify
2021-04-13T01:44:41.5298670Z the server's TLS certificate, or a string, in which case it must be a path
2021-04-13T01:44:41.5299417Z to a CA bundle to use. Defaults to ``True``.
2021-04-13T01:44:41.5300276Z :param stream: (optional) if ``False``, the response content will be immediately downloaded.
2021-04-13T01:44:41.5301531Z :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
2021-04-13T01:44:41.5302375Z :return: :class:`Response <Response>` object
2021-04-13T01:44:41.5303047Z :rtype: requests.Response
2021-04-13T01:44:41.5303517Z
2021-04-13T01:44:41.5304137Z Usage::
2021-04-13T01:44:41.5304509Z
2021-04-13T01:44:41.5304892Z >>> import requests
2021-04-13T01:44:41.5305823Z >>> req = requests.request('GET', 'https://httpbin.org/get')
2021-04-13T01:44:41.5306483Z >>> req
2021-04-13T01:44:41.5306869Z <Response [200]>
2021-04-13T01:44:41.5307265Z """
2021-04-13T01:44:41.5307583Z
2021-04-13T01:44:41.5308386Z # By using the 'with' statement we are sure the session is closed, thus we
2021-04-13T01:44:41.5309288Z # avoid leaving sockets open which can trigger a ResourceWarning in some
2021-04-13T01:44:41.5310081Z # cases, and look like a memory leak in others.
2021-04-13T01:44:41.5310724Z with sessions.Session() as session:
2021-04-13T01:44:41.5311471Z > return session.request(method=method, url=url, **kwargs)
2021-04-13T01:44:41.5311937Z
2021-04-13T01:44:41.5312920Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/requests/api.py:61:
2021-04-13T01:44:41.5313728Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.5314191Z
2021-04-13T01:44:41.5315128Z self = <requests.sessions.Session object at 0x7f6e99ed3350>, method = 'get'
2021-04-13T01:44:41.5316304Z url = 'http://fv-az212-648:8787/status/', params = None, data = None
2021-04-13T01:44:41.5317257Z headers = None, cookies = None, files = None, auth = None, timeout = None
2021-04-13T01:44:41.5318095Z allow_redirects = True, proxies = {}, hooks = None, stream = None, verify = None
2021-04-13T01:44:41.5318754Z cert = None, json = None
2021-04-13T01:44:41.5319040Z
2021-04-13T01:44:41.5319742Z def request(self, method, url,
2021-04-13T01:44:41.5320412Z params=None, data=None, headers=None, cookies=None, files=None,
2021-04-13T01:44:41.5321226Z auth=None, timeout=None, allow_redirects=True, proxies=None,
2021-04-13T01:44:41.5322000Z hooks=None, stream=None, verify=None, cert=None, json=None):
2021-04-13T01:44:41.5322814Z """Constructs a :class:`Request <Request>`, prepares it and sends it.
2021-04-13T01:44:41.5323573Z Returns :class:`Response <Response>` object.
2021-04-13T01:44:41.5324285Z
2021-04-13T01:44:41.5324813Z :param method: method for the new :class:`Request` object.
2021-04-13T01:44:41.5325535Z :param url: URL for the new :class:`Request` object.
2021-04-13T01:44:41.5326275Z :param params: (optional) Dictionary or bytes to be sent in the query
2021-04-13T01:44:41.5326970Z string for the :class:`Request`.
2021-04-13T01:44:41.5328586Z :param data: (optional) Dictionary, list of tuples, bytes, or file-like
2021-04-13T01:44:41.5329416Z object to send in the body of the :class:`Request`.
2021-04-13T01:44:41.5330076Z :param json: (optional) json to send in the body of the
2021-04-13T01:44:41.5330640Z :class:`Request`.
2021-04-13T01:44:41.5331302Z :param headers: (optional) Dictionary of HTTP Headers to send with the
2021-04-13T01:44:41.5331958Z :class:`Request`.
2021-04-13T01:44:41.5332822Z :param cookies: (optional) Dict or CookieJar object to send with the
2021-04-13T01:44:41.5333466Z :class:`Request`.
2021-04-13T01:44:41.5334679Z :param files: (optional) Dictionary of ``'filename': file-like-objects``
2021-04-13T01:44:41.5335457Z for multipart encoding upload.
2021-04-13T01:44:41.5336137Z :param auth: (optional) Auth tuple or callable to enable
2021-04-13T01:44:41.5336779Z Basic/Digest/Custom HTTP Auth.
2021-04-13T01:44:41.5337458Z :param timeout: (optional) How long to wait for the server to send
2021-04-13T01:44:41.5338233Z data before giving up, as a float, or a :ref:`(connect timeout,
2021-04-13T01:44:41.5338938Z read timeout) <timeouts>` tuple.
2021-04-13T01:44:41.5339491Z :type timeout: float or tuple
2021-04-13T01:44:41.5340145Z :param allow_redirects: (optional) Set to True by default.
2021-04-13T01:44:41.5340960Z :type allow_redirects: bool
2021-04-13T01:44:41.5342768Z :param proxies: (optional) Dictionary mapping protocol or protocol and
2021-04-13T01:44:41.5343516Z hostname to the URL of the proxy.
2021-04-13T01:44:41.5344509Z :param stream: (optional) whether to immediately download the response
2021-04-13T01:44:41.5345257Z content. Defaults to ``False``.
2021-04-13T01:44:41.5347685Z :param verify: (optional) Either a boolean, in which case it controls whether we verify
2021-04-13T01:44:41.5348977Z the server's TLS certificate, or a string, in which case it must be a path
2021-04-13T01:44:41.5349760Z to a CA bundle to use. Defaults to ``True``. When set to
2021-04-13T01:44:41.5350535Z ``False``, requests will accept any TLS certificate presented by
2021-04-13T01:44:41.5351404Z the server, and will ignore hostname mismatches and/or expired
2021-04-13T01:44:41.5352268Z certificates, which will make your application vulnerable to
2021-04-13T01:44:41.5353375Z man-in-the-middle (MitM) attacks. Setting verify to ``False``
2021-04-13T01:44:41.5354387Z may be useful during local development or testing.
2021-04-13T01:44:41.5355171Z :param cert: (optional) if String, path to ssl client cert file (.pem).
2021-04-13T01:44:41.5356224Z If Tuple, ('cert', 'key') pair.
2021-04-13T01:44:41.5356785Z :rtype: requests.Response
2021-04-13T01:44:41.5357261Z """
2021-04-13T01:44:41.5357669Z # Create the Request.
2021-04-13T01:44:41.5358128Z req = Request(
2021-04-13T01:44:41.5358607Z method=method.upper(),
2021-04-13T01:44:41.5359074Z url=url,
2021-04-13T01:44:41.5359499Z headers=headers,
2021-04-13T01:44:41.5359949Z files=files,
2021-04-13T01:44:41.5360352Z data=data or {},
2021-04-13T01:44:41.5360753Z json=json,
2021-04-13T01:44:41.5361176Z params=params or {},
2021-04-13T01:44:41.5361593Z auth=auth,
2021-04-13T01:44:41.5362026Z cookies=cookies,
2021-04-13T01:44:41.5362453Z hooks=hooks,
2021-04-13T01:44:41.5362824Z )
2021-04-13T01:44:41.5363300Z prep = self.prepare_request(req)
2021-04-13T01:44:41.5363778Z
2021-04-13T01:44:41.5364346Z proxies = proxies or {}
2021-04-13T01:44:41.5364756Z
2021-04-13T01:44:41.5365274Z settings = self.merge_environment_settings(
2021-04-13T01:44:41.5365978Z prep.url, proxies, stream, verify, cert
2021-04-13T01:44:41.5366470Z )
2021-04-13T01:44:41.5366782Z
2021-04-13T01:44:41.5367173Z # Send the request.
2021-04-13T01:44:41.5367596Z send_kwargs = {
2021-04-13T01:44:41.5368223Z 'timeout': timeout,
2021-04-13T01:44:41.5368930Z 'allow_redirects': allow_redirects,
2021-04-13T01:44:41.5369415Z }
2021-04-13T01:44:41.5370137Z send_kwargs.update(settings)
2021-04-13T01:44:41.5370785Z > resp = self.send(prep, **send_kwargs)
2021-04-13T01:44:41.5371147Z
2021-04-13T01:44:41.5372211Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/requests/sessions.py:542:
2021-04-13T01:44:41.5373274Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.5373580Z
2021-04-13T01:44:41.5374430Z self = <requests.sessions.Session object at 0x7f6e99ed3350>
2021-04-13T01:44:41.5375236Z request = <PreparedRequest [GET]>
2021-04-13T01:44:41.5376269Z kwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': None, ...}
2021-04-13T01:44:41.5377303Z allow_redirects = True, stream = False, hooks = {'response': []}
2021-04-13T01:44:41.5378294Z adapter = <requests.adapters.HTTPAdapter object at 0x7f6e9b0f8690>
2021-04-13T01:44:41.5379055Z start = 1618275495.9322999
2021-04-13T01:44:41.5379325Z
2021-04-13T01:44:41.5379756Z def send(self, request, **kwargs):
2021-04-13T01:44:41.5380506Z """Send a given PreparedRequest.
2021-04-13T01:44:41.5380986Z
2021-04-13T01:44:41.5381462Z :rtype: requests.Response
2021-04-13T01:44:41.5381916Z """
2021-04-13T01:44:41.5382494Z # Set defaults that the hooks can utilize to ensure they always have
2021-04-13T01:44:41.5383342Z # the correct parameters to reproduce the previous request.
2021-04-13T01:44:41.5385571Z kwargs.setdefault('stream', self.stream)
2021-04-13T01:44:41.5386508Z kwargs.setdefault('verify', self.verify)
2021-04-13T01:44:41.5387388Z kwargs.setdefault('cert', self.cert)
2021-04-13T01:44:41.5388307Z kwargs.setdefault('proxies', self.proxies)
2021-04-13T01:44:41.5388853Z
2021-04-13T01:44:41.5389655Z # It's possible that users might accidentally send a Request object.
2021-04-13T01:44:41.5390407Z # Guard against that specific failure case.
2021-04-13T01:44:41.5391043Z if isinstance(request, Request):
2021-04-13T01:44:41.5391972Z raise ValueError('You can only send PreparedRequests.')
2021-04-13T01:44:41.5392574Z
2021-04-13T01:44:41.5393191Z # Set up variables needed for resolve_redirects and dispatching of hooks
2021-04-13T01:44:41.5394404Z allow_redirects = kwargs.pop('allow_redirects', True)
2021-04-13T01:44:41.5395368Z stream = kwargs.get('stream')
2021-04-13T01:44:41.5395908Z hooks = request.hooks
2021-04-13T01:44:41.5396315Z
2021-04-13T01:44:41.5396782Z # Get the appropriate adapter to use
2021-04-13T01:44:41.5397463Z adapter = self.get_adapter(url=request.url)
2021-04-13T01:44:41.5397961Z
2021-04-13T01:44:41.5398467Z # Start time (approximately) of the request
2021-04-13T01:44:41.5399068Z start = preferred_clock()
2021-04-13T01:44:41.5399492Z
2021-04-13T01:44:41.5399864Z # Send the request
2021-04-13T01:44:41.5400407Z > r = adapter.send(request, **kwargs)
2021-04-13T01:44:41.5400773Z
2021-04-13T01:44:41.5401811Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/requests/sessions.py:655:
2021-04-13T01:44:41.5402656Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2021-04-13T01:44:41.5402948Z
2021-04-13T01:44:41.5403677Z self = <requests.adapters.HTTPAdapter object at 0x7f6e9b0f8690>
2021-04-13T01:44:41.5404789Z request = <PreparedRequest [GET]>, stream = False
2021-04-13T01:44:41.5405575Z timeout = Timeout(connect=None, read=None, total=None), verify = True
2021-04-13T01:44:41.5406266Z cert = None, proxies = OrderedDict()
2021-04-13T01:44:41.5406640Z
2021-04-13T01:44:41.5407290Z def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
2021-04-13T01:44:41.5408233Z """Sends PreparedRequest object. Returns Response object.
2021-04-13T01:44:41.5408820Z
2021-04-13T01:44:41.5409527Z :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
2021-04-13T01:44:41.5410466Z :param stream: (optional) Whether to stream the request content.
2021-04-13T01:44:41.5411270Z :param timeout: (optional) How long to wait for the server to send
2021-04-13T01:44:41.5412042Z data before giving up, as a float, or a :ref:`(connect timeout,
2021-04-13T01:44:41.5412954Z read timeout) <timeouts>` tuple.
2021-04-13T01:44:41.5413612Z :type timeout: float or tuple or urllib3 Timeout object
2021-04-13T01:44:41.5414765Z :param verify: (optional) Either a boolean, in which case it controls whether
2021-04-13T01:44:41.5416334Z we verify the server's TLS certificate, or a string, in which case it
2021-04-13T01:44:41.5417066Z must be a path to a CA bundle to use
2021-04-13T01:44:41.5418027Z :param cert: (optional) Any user-provided SSL certificate to be trusted.
2021-04-13T01:44:41.5418975Z :param proxies: (optional) The proxies dictionary to apply to the request.
2021-04-13T01:44:41.5419732Z :rtype: requests.Response
2021-04-13T01:44:41.5420206Z """
2021-04-13T01:44:41.5420899Z
2021-04-13T01:44:41.5421248Z try:
2021-04-13T01:44:41.5421826Z conn = self.get_connection(request.url, proxies)
2021-04-13T01:44:41.5422554Z except LocationValueError as e:
2021-04-13T01:44:41.5423248Z raise InvalidURL(e, request=request)
2021-04-13T01:44:41.5423729Z
2021-04-13T01:44:41.5424274Z self.cert_verify(conn, request.url, verify, cert)
2021-04-13T01:44:41.5424984Z url = self.request_url(request, proxies)
2021-04-13T01:44:41.5426243Z self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
2021-04-13T01:44:41.5426977Z
2021-04-13T01:44:41.5427945Z chunked = not (request.body is None or 'Content-Length' in request.headers)
2021-04-13T01:44:41.5428619Z
2021-04-13T01:44:41.5429076Z if isinstance(timeout, tuple):
2021-04-13T01:44:41.5429529Z try:
2021-04-13T01:44:41.5429951Z connect, read = timeout
2021-04-13T01:44:41.5430619Z timeout = TimeoutSauce(connect=connect, read=read)
2021-04-13T01:44:41.5431281Z except ValueError as e:
2021-04-13T01:44:41.5431907Z # this may raise a string formatting error.
2021-04-13T01:44:41.5432564Z err = ("Invalid timeout {}. Pass a (connect, read) "
2021-04-13T01:44:41.5433383Z "timeout tuple, or a single float to set "
2021-04-13T01:44:41.5434049Z "both timeouts to the same value".format(timeout))
2021-04-13T01:44:41.5434667Z raise ValueError(err)
2021-04-13T01:44:41.5435318Z elif isinstance(timeout, TimeoutSauce):
2021-04-13T01:44:41.5435853Z pass
2021-04-13T01:44:41.5436211Z else:
2021-04-13T01:44:41.5436799Z timeout = TimeoutSauce(connect=timeout, read=timeout)
2021-04-13T01:44:41.5437352Z
2021-04-13T01:44:41.5437686Z try:
2021-04-13T01:44:41.5438068Z if not chunked:
2021-04-13T01:44:41.5438541Z resp = conn.urlopen(
2021-04-13T01:44:41.5439101Z method=request.method,
2021-04-13T01:44:41.5439583Z url=url,
2021-04-13T01:44:41.5440043Z body=request.body,
2021-04-13T01:44:41.5440605Z headers=request.headers,
2021-04-13T01:44:41.5441161Z redirect=False,
2021-04-13T01:44:41.5441652Z assert_same_host=False,
2021-04-13T01:44:41.5442176Z preload_content=False,
2021-04-13T01:44:41.5442685Z decode_content=False,
2021-04-13T01:44:41.5443234Z retries=self.max_retries,
2021-04-13T01:44:41.5443744Z timeout=timeout
2021-04-13T01:44:41.5444147Z )
2021-04-13T01:44:41.5444466Z
2021-04-13T01:44:41.5444849Z # Send the request.
2021-04-13T01:44:41.5445269Z else:
2021-04-13T01:44:41.5445921Z if hasattr(conn, 'proxy_pool'):
2021-04-13T01:44:41.5446468Z conn = conn.proxy_pool
2021-04-13T01:44:41.5446875Z
2021-04-13T01:44:41.5447415Z low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
2021-04-13T01:44:41.5447930Z
2021-04-13T01:44:41.5448271Z try:
2021-04-13T01:44:41.5448818Z low_conn.putrequest(request.method,
2021-04-13T01:44:41.5449397Z url,
2021-04-13T01:44:41.5449891Z skip_accept_encoding=True)
2021-04-13T01:44:41.5450349Z
2021-04-13T01:44:41.5450926Z for header, value in request.headers.items():
2021-04-13T01:44:41.5451691Z low_conn.putheader(header, value)
2021-04-13T01:44:41.5452184Z
2021-04-13T01:44:41.5452885Z low_conn.endheaders()
2021-04-13T01:44:41.5453339Z
2021-04-13T01:44:41.5453753Z for i in request.body:
2021-04-13T01:44:41.5454690Z low_conn.send(hex(len(i))[2:].encode('utf-8'))
2021-04-13T01:44:41.5455446Z low_conn.send(b'\r\n')
2021-04-13T01:44:41.5455939Z low_conn.send(i)
2021-04-13T01:44:41.5456581Z low_conn.send(b'\r\n')
2021-04-13T01:44:41.5457256Z low_conn.send(b'0\r\n\r\n')
2021-04-13T01:44:41.5457668Z
2021-04-13T01:44:41.5458167Z # Receive the response from the server
2021-04-13T01:44:41.5458660Z try:
2021-04-13T01:44:41.5459204Z # For Python 2.7, use buffering of HTTP responses
2021-04-13T01:44:41.5459937Z r = low_conn.getresponse(buffering=True)
2021-04-13T01:44:41.5460555Z except TypeError:
2021-04-13T01:44:41.5461137Z # For compatibility with Python 3.3+
2021-04-13T01:44:41.5461753Z r = low_conn.getresponse()
2021-04-13T01:44:41.5462217Z
2021-04-13T01:44:41.5462731Z resp = HTTPResponse.from_httplib(
2021-04-13T01:44:41.5463271Z r,
2021-04-13T01:44:41.5463656Z pool=conn,
2021-04-13T01:44:41.5464135Z connection=low_conn,
2021-04-13T01:44:41.5464653Z preload_content=False,
2021-04-13T01:44:41.5465269Z decode_content=False
2021-04-13T01:44:41.5465686Z )
2021-04-13T01:44:41.5466065Z except:
2021-04-13T01:44:41.5466637Z # If we hit any problems here, clean up the connection.
2021-04-13T01:44:41.5467403Z # Then, reraise so that we can handle the actual exception.
2021-04-13T01:44:41.5468040Z low_conn.close()
2021-04-13T01:44:41.5468457Z raise
2021-04-13T01:44:41.5468809Z
2021-04-13T01:44:41.5469345Z except (ProtocolError, socket.error) as err:
2021-04-13T01:44:41.5470120Z raise ConnectionError(err, request=request)
2021-04-13T01:44:41.5470648Z
2021-04-13T01:44:41.5471336Z except MaxRetryError as e:
2021-04-13T01:44:41.5472076Z if isinstance(e.reason, ConnectTimeoutError):
2021-04-13T01:44:41.5472784Z # TODO: Remove this in 3.0.0: see #2811
2021-04-13T01:44:41.5473470Z if not isinstance(e.reason, NewConnectionError):
2021-04-13T01:44:41.5474275Z raise ConnectTimeout(e, request=request)
2021-04-13T01:44:41.5474795Z
2021-04-13T01:44:41.5475313Z if isinstance(e.reason, ResponseError):
2021-04-13T01:44:41.5475991Z raise RetryError(e, request=request)
2021-04-13T01:44:41.5476468Z
2021-04-13T01:44:41.5476965Z if isinstance(e.reason, _ProxyError):
2021-04-13T01:44:41.5477620Z raise ProxyError(e, request=request)
2021-04-13T01:44:41.5478110Z
2021-04-13T01:44:41.5478574Z if isinstance(e.reason, _SSLError):
2021-04-13T01:44:41.5492228Z # This branch is for urllib3 v1.22 and later.
2021-04-13T01:44:41.5493128Z raise SSLError(e, request=request)
2021-04-13T01:44:41.5493612Z
2021-04-13T01:44:41.5494134Z > raise ConnectionError(e, request=request)
2021-04-13T01:44:41.5496984Z E requests.exceptions.ConnectionError: HTTPConnectionPool(host='fv-az212-648', port=8787): Max retries exceeded with url: /status/ (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f6e9a188310>: Failed to establish a new connection: [Errno 110] Connection timed out'))
2021-04-13T01:44:41.5498895Z
2021-04-13T01:44:41.5500039Z /usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/requests/adapters.py:516: ConnectionError
2021-04-13T01:44:41.5500831Z
2021-04-13T01:44:41.5501456Z During handling of the above exception, another exception occurred:
2021-04-13T01:44:41.5501963Z
2021-04-13T01:44:41.5502805Z loop = <tornado.platform.asyncio.AsyncIOLoop object at 0x7f6e99dd7e90>
2021-04-13T01:44:41.5503555Z
2021-04-13T01:44:41.5504105Z def test_dashboard(loop):
2021-04-13T01:44:41.5504694Z pytest.importorskip("bokeh")
2021-04-13T01:44:41.5505196Z
2021-04-13T01:44:41.5505869Z with popen(["dask-scheduler"]) as proc:
2021-04-13T01:44:41.5506469Z for line in proc.stderr:
2021-04-13T01:44:41.5506995Z if b"dashboard at" in line:
2021-04-13T01:44:41.5507843Z dashboard_port = int(line.decode().split(":")[-1].strip())
2021-04-13T01:44:41.5508402Z break
2021-04-13T01:44:41.5508785Z else:
2021-04-13T01:44:41.5509288Z raise Exception("dashboard not found")
2021-04-13T01:44:41.5509772Z
2021-04-13T01:44:41.5510344Z with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
2021-04-13T01:44:41.5510954Z pass
2021-04-13T01:44:41.5511290Z
2021-04-13T01:44:41.5511716Z names = ["localhost", "127.0.0.1", get_ip()]
2021-04-13T01:44:41.5512517Z if "linux" in sys.platform:
2021-04-13T01:44:41.5513196Z names.append(socket.gethostname())
2021-04-13T01:44:41.5513743Z
2021-04-13T01:44:41.5514099Z start = time()
2021-04-13T01:44:41.5514518Z while True:
2021-04-13T01:44:41.5514887Z try:
2021-04-13T01:44:41.5515463Z # All addresses should respond
2021-04-13T01:44:41.5515974Z for name in names:
2021-04-13T01:44:41.5516584Z uri = "http://%s:%d/status/" % (name, dashboard_port)
2021-04-13T01:44:41.5517242Z response = requests.get(uri)
2021-04-13T01:44:41.5517822Z assert response.ok
2021-04-13T01:44:41.5518263Z break
2021-04-13T01:44:41.5518727Z except Exception as f:
2021-04-13T01:44:41.5519257Z print("got error on %r: %s" % (uri, f))
2021-04-13T01:44:41.5519738Z sleep(0.1)
2021-04-13T01:44:41.5520185Z > assert time() < start + 10
2021-04-13T01:44:41.5520668Z E assert 1618275625.7580516 < (1618275495.6326065 + 10)
2021-04-13T01:44:41.5521157Z E + where 1618275625.7580516 = time()
2021-04-13T01:44:41.5521469Z
2021-04-13T01:44:41.5522334Z distributed/cli/tests/test_dask_scheduler.py:95: AssertionError
2021-04-13T01:44:41.5523505Z ----------------------------- Captured stdout call -----------------------------
2021-04-13T01:44:41.5525987Z got error on 'http://fv-az212-648:8787/status/': HTTPConnectionPool(host='fv-az212-648', port=8787): Max retries exceeded with url: /status/ (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f6e9a188310>: Failed to establish a new connection: [Errno 110] Connection timed out'))
2021-04-13T01:44:41.5527685Z
2021-04-13T01:44:41.5527920Z
2021-04-13T01:44:41.5528281Z Print from stderr
2021-04-13T01:44:41.5528663Z /
2021-04-13T01:44:41.5528976Z =================
2021-04-13T01:44:41.5529219Z
2021-04-13T01:44:41.5530422Z distributed.scheduler - INFO - Receive client connection: Client-5426c198-9bf3-11eb-8869-9592fccfcded
2021-04-13T01:44:41.5531828Z distributed.core - INFO - Starting established connection
2021-04-13T01:44:41.5533354Z distributed.scheduler - INFO - Remove client Client-5426c198-9bf3-11eb-8869-9592fccfcded
2021-04-13T01:44:41.5535022Z distributed.scheduler - INFO - Remove client Client-5426c198-9bf3-11eb-8869-9592fccfcded
2021-04-13T01:44:41.5536773Z distributed.scheduler - INFO - Close client connection: Client-5426c198-9bf3-11eb-8869-9592fccfcded
2021-04-13T01:44:41.5538255Z distributed.scheduler - INFO - End scheduler at 'tcp://10.1.0.130:8786'
2021-04-13T01:44:41.5539050Z Traceback (most recent call last):
2021-04-13T01:44:41.5540106Z File "/usr/share/miniconda3/envs/dask-distributed/bin/dask-scheduler", line 33, in <module>
2021-04-13T01:44:41.5541356Z sys.exit(load_entry_point('distributed', 'console_scripts', 'dask-scheduler')())
2021-04-13T01:44:41.5542554Z File "/home/runner/work/distributed/distributed/distributed/cli/dask_scheduler.py", line 218, in go
2021-04-13T01:44:41.5543301Z main()
2021-04-13T01:44:41.5544388Z File "/usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/click/core.py", line 829, in __call__
2021-04-13T01:44:41.5545303Z return self.main(*args, **kwargs)
2021-04-13T01:44:41.5546476Z File "/usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/click/core.py", line 782, in main
2021-04-13T01:44:41.5547337Z rv = self.invoke(ctx)
2021-04-13T01:44:41.5548478Z File "/usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
2021-04-13T01:44:41.5549491Z return ctx.invoke(self.callback, **ctx.params)
2021-04-13T01:44:41.5550758Z File "/usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/click/core.py", line 610, in invoke
2021-04-13T01:44:41.5551669Z return callback(*args, **kwargs)
2021-04-13T01:44:41.5552534Z File "/home/runner/work/distributed/distributed/distributed/cli/dask_scheduler.py", line 209, in main
2021-04-13T01:44:41.5553320Z loop.run_sync(run)
2021-04-13T01:44:41.5554465Z File "/usr/share/miniconda3/envs/dask-distributed/lib/python3.7/site-packages/tornado/ioloop.py", line 575, in run_sync
2021-04-13T01:44:41.5555755Z raise TimeoutError('Operation timed out after %s seconds' % timeout)
2021-04-13T01:44:41.5556840Z tornado.util.TimeoutError: Operation timed out after None seconds
2021-04-13T01:44:41.5557436Z
2021-04-13T01:44:41.5557652Z
2021-04-13T01:44:41.5557873Z
2021-04-13T01:44:41.5558244Z Print from stdout
2021-04-13T01:44:41.5558618Z =================
2021-04-13T01:44:41.5558857Z
2021-04-13T01:44:41.5559084Z