Search and Agent search not ok

[tkfischer]

Hi,

Trying to get chat working per #4319 results in more issues. Seems like Vespa is not connecting per trace below. Also, the app goes into a irrecoverable state with error msg even for new chat. Have tried a few times now with re-create, re-install and restart everything including laptop. I am using exact distribution and docker deployment on a MacBookPro. It was working two days back.

Thanks,
Tim

"Invalid Chat Message specified"

Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", line 69, in map_httpcore_exceptions
    yield
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", line 233, in handle_request
    resp = self._pool.handle_request(req)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py", line 216, in handle_request
    raise exc from None
  File "/usr/local/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py", line 196, in handle_request
    response = connection.handle_request(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpcore/_sync/connection.py", line 99, in handle_request
    raise exc
  File "/usr/local/lib/python3.11/site-packages/httpcore/_sync/connection.py", line 76, in handle_request
    stream = self._connect(request)
             ^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpcore/_sync/connection.py", line 122, in _connect
    stream = self._network_backend.connect_tcp(**kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpcore/_backends/sync.py", line 205, in connect_tcp
    with map_exceptions(exc_map):
  File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.11/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
    raise to_exc(exc) from exc
httpcore.ConnectError: [Errno 111] Connection refused

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/app/onyx/document_index/vespa/chunk_retrieval.py", line 306, in query_vespa
    response = http_client.post(SEARCH_ENDPOINT, json=params)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1145, in post
    return self.request(
           ^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 827, in request
    return self.send(request, auth=auth, follow_redirects=follow_redirects)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 914, in send
    response = self._send_handling_auth(
               ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 942, in _send_handling_auth
    response = self._send_handling_redirects(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 979, in _send_handling_redirects
    response = self._send_single_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1015, in _send_single_request
    response = transport.handle_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", line 232, in handle_request
    with map_httpcore_exceptions():
  File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", line 86, in map_httpcore_exceptions
    raise mapped_exc(message) from exc
httpx.ConnectError: [Errno 111] Connection refused

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/app/onyx/chat/process_message.py", line 799, in stream_chat_message_objects
    for packet in answer.processed_streamed_output:
  File "/app/onyx/chat/answer.py", line 153, in processed_streamed_output
    for packet in stream:
  File "/app/onyx/agents/agent_search/run_graph.py", line 137, in run_main_graph
    yield from run_graph(compiled_graph, config, input)
  File "/app/onyx/agents/agent_search/run_graph.py", line 106, in run_graph
    for event in manage_sync_streaming(
  File "/app/onyx/agents/agent_search/run_graph.py", line 88, in manage_sync_streaming
    for event in compiled_graph.stream(
  File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/__init__.py", line 1724, in stream
    for _ in runner.tick(
  File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/runner.py", line 302, in tick
    _panic_or_proceed(
  File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/runner.py", line 619, in _panic_or_proceed
    raise exc
  File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/executor.py", line 83, in done
    task.result()
  File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/retry.py", line 40, in run_with_retry
    return task.proc.invoke(task.input, config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/langgraph/utils/runnable.py", line 506, in invoke
    input = step.invoke(input, config, **kwargs)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/langgraph/utils/runnable.py", line 270, in invoke
    ret = context.run(self.func, *args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/agents/agent_search/deep_search/main/nodes/start_agent_search.py", line 38, in start_agent_search
    retrieved_docs: list[InferenceSection] = retrieve_search_docs(search_tool, question)
                                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/agents/agent_search/shared_graph_utils/utils.py", line 364, in retrieve_search_docs
    for tool_response in search_tool.run(
  File "/app/onyx/tools/tool_implementations/search/search_tool.py", line 359, in run
    yield from yield_search_responses(
  File "/app/onyx/tools/tool_implementations/search/search_tool.py", line 418, in yield_search_responses
    top_sections=get_retrieved_sections(),
                 ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/tools/tool_implementations/search/search_tool.py", line 361, in <lambda>
    lambda: search_pipeline.retrieved_sections,
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/context/search/pipeline.py", line 339, in retrieved_sections
    self._retrieved_sections = self._get_sections()
                               ^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/utils/timing.py", line 31, in wrapped_func
    result = func(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/context/search/pipeline.py", line 179, in _get_sections
    retrieved_chunks = self._get_chunks()
                       ^^^^^^^^^^^^^^^^^^
  File "/app/onyx/context/search/pipeline.py", line 152, in _get_chunks
    self._retrieved_chunks = retrieve_chunks(
                             ^^^^^^^^^^^^^^^^
  File "/app/onyx/context/search/retrieval/search_runner.py", line 238, in retrieve_chunks
    top_chunks = doc_index_retrieval(
                 ^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/utils/timing.py", line 31, in wrapped_func
    result = func(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/context/search/retrieval/search_runner.py", line 142, in doc_index_retrieval
    top_chunks = document_index.hybrid_retrieval(
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/onyx/document_index/vespa/index.py", line 815, in hybrid_retrieval
    return query_vespa(params)
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/decorator.py", line 235, in fun
    return caller(func, *(extras + args), **kw)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/retry/api.py", line 73, in retry_decorator
    return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/retry/api.py", line 33, in __retry_internal
    return f()
           ^^^
  File "/app/onyx/document_index/vespa/chunk_retrieval.py", line 322, in query_vespa
    raise httpx.HTTPError(error_base) from e
httpx.HTTPError: Failed to query Vespa
During task with name 'start_agent_search' and id '89d35bc7-b9ae-b379-d7f5-0c3185f021a5'