Page MenuHomePhabricator
Paste P65221

(An Untitled Masterwork)
ActivePublic
Actions

Authored by isarantopoulos on Thu, Jun 20, 9:52 AM.
Tags
None
Referenced Files
F55490644: raw-paste-data.txt
Thu, Jun 20, 9:52 AM
Subscribers
None
```
kubectl logs -f llama3-predictor-00009-deployment-6c6cf64695-v2c7v storage-initializer
INFO:root:Initializing, args: src_uri [s3://wmf-ml-models/llm/llama3-8B-instruct/] dest_path[ [/mnt/models]
INFO:root:Copying contents of s3://wmf-ml-models/llm/llama3-8B-instruct/ to local
INFO:botocore.credentials:Found credentials in environment variables.
Traceback (most recent call last):
File "/kserve/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/socket.py", line 962, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
socket.gaierror: [Errno -3] Temporary failure in name resolution
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/kserve/site-packages/botocore/httpsession.py", line 464, in send
urllib_response = conn.urlopen(
^^^^^^^^^^^^^
File "/kserve/site-packages/urllib3/connectionpool.py", line 799, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/urllib3/util/retry.py", line 525, in increment
raise six.reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/urllib3/packages/six.py", line 770, in reraise
raise value
File "/kserve/site-packages/urllib3/connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/kserve/site-packages/urllib3/connectionpool.py", line 1058, in _validate_conn
conn.connect()
File "/kserve/site-packages/urllib3/connection.py", line 363, in connect
self.sock = conn = self._new_conn()
^^^^^^^^^^^^^^^^
File "/kserve/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <botocore.awsrequest.AWSHTTPSConnection object at 0x7f1e7aeb9690>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/bin/storage-initializer-entrypoint", line 15, in <module>
Storage.download(src_uri, dest_path)
File "/kserve/site-packages/kserve/storage/storage.py", line 83, in download
Storage._download_s3(uri, out_dir)
File "/kserve/site-packages/kserve/storage/storage.py", line 177, in _download_s3
for obj in bucket.objects.filter(Prefix=bucket_path):
File "/kserve/site-packages/boto3/resources/collection.py", line 81, in __iter__
for page in self.pages():
File "/kserve/site-packages/boto3/resources/collection.py", line 171, in pages
for page in pages:
File "/kserve/site-packages/botocore/paginate.py", line 269, in __iter__
response = self._make_request(current_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/paginate.py", line 357, in _make_request
return self._method(**current_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/client.py", line 553, in _api_call
return self._make_api_call(operation_name, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/client.py", line 989, in _make_api_call
http, parsed_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/client.py", line 1015, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/endpoint.py", line 119, in make_request
return self._send_request(request_dict, operation_model)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/endpoint.py", line 202, in _send_request
while self._needs_retry(
^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/endpoint.py", line 354, in _needs_retry
responses = self._event_emitter.emit(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/hooks.py", line 412, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/hooks.py", line 256, in emit
return self._emit(event_name, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/hooks.py", line 239, in _emit
response = handler(**kwargs)
^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/retryhandler.py", line 207, in __call__
if self._checker(**checker_kwargs):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/retryhandler.py", line 284, in __call__
should_retry = self._should_retry(
^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/retryhandler.py", line 320, in _should_retry
return self._checker(attempt_number, response, caught_exception)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/retryhandler.py", line 363, in __call__
checker_response = checker(
^^^^^^^^
File "/kserve/site-packages/botocore/retryhandler.py", line 247, in __call__
return self._check_caught_exception(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/retryhandler.py", line 416, in _check_caught_exception
raise caught_exception
File "/kserve/site-packages/botocore/endpoint.py", line 281, in _do_get_response
http_response = self._send(request)
^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/endpoint.py", line 377, in _send
return self.http_session.send(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/kserve/site-packages/botocore/httpsession.py", line 493, in send
raise EndpointConnectionError(endpoint_url=request.url, error=e)
botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "https://thanos-swift.discovery.wmnet/wmf-ml-models?prefix=llm%2Fllama3-8B-instruct%2F&encoding-type=url"
```
Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits