Swarms Test Run Report

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 108, in run_long_query
    assert result.exitcode in expected_exitcodes, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Given)
  def run_long_query(
      self,
      node,
      clickhouse_iceberg_table_name,
      cluster_name="static_swarm_cluster",
      sleep_each_row=1,
      exitcode=None,
      message=None,
      expected_exitcodes=None,
      expected_messages=None,
      delay_before_execution=None,
      expected_result=None,
      max_threads=1,
      lock_object_storage_task_distribution_ms=None,
  ):
      """Run a long select from an iceberg table.

      When ``expected_exitcodes`` is provided the query is run without the single
      exit code / message assertions and the result is instead checked against an
      allow-list of acceptable outcomes. This is used when the failure is
      inherently racy: stopping a swarm node mid-query can surface either a
      graceful cancellation or an abrupt connection drop, and both are valid.
      """
      if delay_before_execution:
          time.sleep(delay_before_execution)

      use_allow_list = expected_exitcodes is not None

      result = node.query(
          f"""
              SELECT count(), hostName() 
              FROM {clickhouse_iceberg_table_name} 
              WHERE NOT ignore(sleepEachRow({sleep_each_row})) 
              GROUP BY hostName()
              SETTINGS 
                  object_storage_cluster='{cluster_name}', 
                  max_threads={max_threads}
                  {f", lock_object_storage_task_distribution_ms={lock_object_storage_task_distribution_ms}" if lock_object_storage_task_distribution_ms is not None else ""}
          """,
          exitcode=None if use_allow_list else exitcode,
          message=None if use_allow_list else message,
          no_checks=use_allow_list,
      )
      note(f"RESULT: \n{result.output}\n")

      if use_allow_list:
          assert result.exitcode in expected_exitcodes, error(

Description
  Expected exit code in [138], but got: 0

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py', line 108 in 'run_long_query'

100\|          """,
101\|          exitcode=None if use_allow_list else exitcode,
102\|          message=None if use_allow_list else message,
103\|          no_checks=use_allow_list,
104\|      )
105\|      note(f"RESULT: \n{result.output}\n")
106\|  
107\|      if use_allow_list:
108\|>         assert result.exitcode in expected_exitcodes, error(
109\|              f"Expected exit code in {expected_exitcodes}, but got: {result.exitcode}"
110\|          )
111\|          if expected_messages:

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 108, in run_long_query
    assert result.exitcode in expected_exitcodes, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Given)
  def run_long_query(
      self,
      node,
      clickhouse_iceberg_table_name,
      cluster_name="static_swarm_cluster",
      sleep_each_row=1,
      exitcode=None,
      message=None,
      expected_exitcodes=None,
      expected_messages=None,
      delay_before_execution=None,
      expected_result=None,
      max_threads=1,
      lock_object_storage_task_distribution_ms=None,
  ):
      """Run a long select from an iceberg table.

      When ``expected_exitcodes`` is provided the query is run without the single
      exit code / message assertions and the result is instead checked against an
      allow-list of acceptable outcomes. This is used when the failure is
      inherently racy: stopping a swarm node mid-query can surface either a
      graceful cancellation or an abrupt connection drop, and both are valid.
      """
      if delay_before_execution:
          time.sleep(delay_before_execution)

      use_allow_list = expected_exitcodes is not None

      result = node.query(
          f"""
              SELECT count(), hostName() 
              FROM {clickhouse_iceberg_table_name} 
              WHERE NOT ignore(sleepEachRow({sleep_each_row})) 
              GROUP BY hostName()
              SETTINGS 
                  object_storage_cluster='{cluster_name}', 
                  max_threads={max_threads}
                  {f", lock_object_storage_task_distribution_ms={lock_object_storage_task_distribution_ms}" if lock_object_storage_task_distribution_ms is not None else ""}
          """,
          exitcode=None if use_allow_list else exitcode,
          message=None if use_allow_list else message,
          no_checks=use_allow_list,
      )
      note(f"RESULT: \n{result.output}\n")

      if use_allow_list:
          assert result.exitcode in expected_exitcodes, error(

Description
  Expected exit code in [138], but got: 0

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py', line 108 in 'run_long_query'

100\|          """,
101\|          exitcode=None if use_allow_list else exitcode,
102\|          message=None if use_allow_list else message,
103\|          no_checks=use_allow_list,
104\|      )
105\|      note(f"RESULT: \n{result.output}\n")
106\|  
107\|      if use_allow_list:
108\|>         assert result.exitcode in expected_exitcodes, error(
109\|              f"Expected exit code in {expected_exitcodes}, but got: {result.exitcode}"
110\|          )
111\|          if expected_messages:

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 108, in run_long_query
    assert result.exitcode in expected_exitcodes, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Given)
  def run_long_query(
      self,
      node,
      clickhouse_iceberg_table_name,
      cluster_name="static_swarm_cluster",
      sleep_each_row=1,
      exitcode=None,
      message=None,
      expected_exitcodes=None,
      expected_messages=None,
      delay_before_execution=None,
      expected_result=None,
      max_threads=1,
      lock_object_storage_task_distribution_ms=None,
  ):
      """Run a long select from an iceberg table.

      When ``expected_exitcodes`` is provided the query is run without the single
      exit code / message assertions and the result is instead checked against an
      allow-list of acceptable outcomes. This is used when the failure is
      inherently racy: stopping a swarm node mid-query can surface either a
      graceful cancellation or an abrupt connection drop, and both are valid.
      """
      if delay_before_execution:
          time.sleep(delay_before_execution)

      use_allow_list = expected_exitcodes is not None

      result = node.query(
          f"""
              SELECT count(), hostName() 
              FROM {clickhouse_iceberg_table_name} 
              WHERE NOT ignore(sleepEachRow({sleep_each_row})) 
              GROUP BY hostName()
              SETTINGS 
                  object_storage_cluster='{cluster_name}', 
                  max_threads={max_threads}
                  {f", lock_object_storage_task_distribution_ms={lock_object_storage_task_distribution_ms}" if lock_object_storage_task_distribution_ms is not None else ""}
          """,
          exitcode=None if use_allow_list else exitcode,
          message=None if use_allow_list else message,
          no_checks=use_allow_list,
      )
      note(f"RESULT: \n{result.output}\n")

      if use_allow_list:
          assert result.exitcode in expected_exitcodes, error(

Description
  Expected exit code in [138], but got: 0

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py', line 108 in 'run_long_query'

100\|          """,
101\|          exitcode=None if use_allow_list else exitcode,
102\|          message=None if use_allow_list else message,
103\|          no_checks=use_allow_list,
104\|      )
105\|      note(f"RESULT: \n{result.output}\n")
106\|  
107\|      if use_allow_list:
108\|>         assert result.exitcode in expected_exitcodes, error(
109\|              f"Expected exit code in {expected_exitcodes}, but got: {result.exitcode}"
110\|          )
111\|          if expected_messages:

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 108, in run_long_query
    assert result.exitcode in expected_exitcodes, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Given)
  def run_long_query(
      self,
      node,
      clickhouse_iceberg_table_name,
      cluster_name="static_swarm_cluster",
      sleep_each_row=1,
      exitcode=None,
      message=None,
      expected_exitcodes=None,
      expected_messages=None,
      delay_before_execution=None,
      expected_result=None,
      max_threads=1,
      lock_object_storage_task_distribution_ms=None,
  ):
      """Run a long select from an iceberg table.

      When ``expected_exitcodes`` is provided the query is run without the single
      exit code / message assertions and the result is instead checked against an
      allow-list of acceptable outcomes. This is used when the failure is
      inherently racy: stopping a swarm node mid-query can surface either a
      graceful cancellation or an abrupt connection drop, and both are valid.
      """
      if delay_before_execution:
          time.sleep(delay_before_execution)

      use_allow_list = expected_exitcodes is not None

      result = node.query(
          f"""
              SELECT count(), hostName() 
              FROM {clickhouse_iceberg_table_name} 
              WHERE NOT ignore(sleepEachRow({sleep_each_row})) 
              GROUP BY hostName()
              SETTINGS 
                  object_storage_cluster='{cluster_name}', 
                  max_threads={max_threads}
                  {f", lock_object_storage_task_distribution_ms={lock_object_storage_task_distribution_ms}" if lock_object_storage_task_distribution_ms is not None else ""}
          """,
          exitcode=None if use_allow_list else exitcode,
          message=None if use_allow_list else message,
          no_checks=use_allow_list,
      )
      note(f"RESULT: \n{result.output}\n")

      if use_allow_list:
          assert result.exitcode in expected_exitcodes, error(

Description
  Expected exit code in [138], but got: 0

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py', line 108 in 'run_long_query'

100\|          """,
101\|          exitcode=None if use_allow_list else exitcode,
102\|          message=None if use_allow_list else message,
103\|          no_checks=use_allow_list,
104\|      )
105\|      note(f"RESULT: \n{result.output}\n")
106\|  
107\|      if use_allow_list:
108\|>         assert result.exitcode in expected_exitcodes, error(
109\|              f"Expected exit code in {expected_exitcodes}, but got: {result.exitcode}"
110\|          )
111\|          if expected_messages: