Swarms Test Run Report

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 32, in run_long_query
    result = node.query(
             ^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py", line 1242, in query
    assert r.exitcode == exitcode, error(r.output)
           ^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert r.exitcode == exitcode, error(r.output)

Description
  Received exception from server (version 25.8.14):
Code: 32. DB::Exception: Received from localhost:9000. DB::Exception: Attempt to read after eof: while receiving packet from clickhouse2:9000, 172.18.0.8, local address: 172.18.0.10:59382: While executing Remote. (ATTEMPT_TO_READ_AFTER_EOF)
(query: SELECT count(), hostName() 
            FROM datalakecatalog_db_0ea1b46f_fd49_11f0_a4fe_9200071779f7.`namespace_0ea1c2ea_fd49_11f0_8e6f_9200071779f7.table_0ea1c34b_fd49_11f0_8a68_9200071779f7` 
            WHERE NOT ignore(sleepEachRow(1)) 
            GROUP BY hostName()
            SETTINGS 
                object_storage_cluster='static_swarm_cluster', 
                max_threads=1


)

Assertion values
  assert r.exitcode == exitcode, error(r.output)
         ^ is 
  assert r.exitcode == exitcode, error(r.output)
         ^ is = 32
  assert r.exitcode == exitcode, error(r.output)
                       ^ is 138
  assert r.exitcode == exitcode, error(r.output)
                    ^ is = False
  assert r.exitcode == exitcode, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py', line 1242 in 'query'

1234\|                      **kwargs,
1235\|                  )
1236\|  
1237\|          if no_checks:
1238\|              return r
1239\|  
1240\|          if exitcode is not None:
1241\|              with Then(f"exitcode should be {exitcode}") if steps else NullStep():
1242\|>                 assert r.exitcode == exitcode, error(r.output)
1243\|  
1244\|          if message is not None:
1245\|              with (

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 32, in run_long_query
    result = node.query(
             ^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py", line 1242, in query
    assert r.exitcode == exitcode, error(r.output)
           ^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert r.exitcode == exitcode, error(r.output)

Description
  Received exception from server (version 25.8.14):
Code: 32. DB::Exception: Received from localhost:9000. DB::Exception: Attempt to read after eof: while receiving packet from clickhouse2:9000, 172.18.0.8, local address: 172.18.0.10:59382: While executing Remote. (ATTEMPT_TO_READ_AFTER_EOF)
(query: SELECT count(), hostName() 
            FROM datalakecatalog_db_0ea1b46f_fd49_11f0_a4fe_9200071779f7.`namespace_0ea1c2ea_fd49_11f0_8e6f_9200071779f7.table_0ea1c34b_fd49_11f0_8a68_9200071779f7` 
            WHERE NOT ignore(sleepEachRow(1)) 
            GROUP BY hostName()
            SETTINGS 
                object_storage_cluster='static_swarm_cluster', 
                max_threads=1


)

Assertion values
  assert r.exitcode == exitcode, error(r.output)
         ^ is 
  assert r.exitcode == exitcode, error(r.output)
         ^ is = 32
  assert r.exitcode == exitcode, error(r.output)
                       ^ is 138
  assert r.exitcode == exitcode, error(r.output)
                    ^ is = False
  assert r.exitcode == exitcode, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py', line 1242 in 'query'

1234\|                      **kwargs,
1235\|                  )
1236\|  
1237\|          if no_checks:
1238\|              return r
1239\|  
1240\|          if exitcode is not None:
1241\|              with Then(f"exitcode should be {exitcode}") if steps else NullStep():
1242\|>                 assert r.exitcode == exitcode, error(r.output)
1243\|  
1244\|          if message is not None:
1245\|              with (

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 32, in run_long_query
    result = node.query(
             ^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py", line 1242, in query
    assert r.exitcode == exitcode, error(r.output)
           ^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert r.exitcode == exitcode, error(r.output)

Description
  Received exception from server (version 25.8.14):
Code: 32. DB::Exception: Received from localhost:9000. DB::Exception: Attempt to read after eof: while receiving packet from clickhouse2:9000, 172.18.0.8, local address: 172.18.0.10:59382: While executing Remote. (ATTEMPT_TO_READ_AFTER_EOF)
(query: SELECT count(), hostName() 
            FROM datalakecatalog_db_0ea1b46f_fd49_11f0_a4fe_9200071779f7.`namespace_0ea1c2ea_fd49_11f0_8e6f_9200071779f7.table_0ea1c34b_fd49_11f0_8a68_9200071779f7` 
            WHERE NOT ignore(sleepEachRow(1)) 
            GROUP BY hostName()
            SETTINGS 
                object_storage_cluster='static_swarm_cluster', 
                max_threads=1


)

Assertion values
  assert r.exitcode == exitcode, error(r.output)
         ^ is 
  assert r.exitcode == exitcode, error(r.output)
         ^ is = 32
  assert r.exitcode == exitcode, error(r.output)
                       ^ is 138
  assert r.exitcode == exitcode, error(r.output)
                    ^ is = False
  assert r.exitcode == exitcode, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py', line 1242 in 'query'

1234\|                      **kwargs,
1235\|                  )
1236\|  
1237\|          if no_checks:
1238\|              return r
1239\|  
1240\|          if exitcode is not None:
1241\|              with Then(f"exitcode should be {exitcode}") if steps else NullStep():
1242\|>                 assert r.exitcode == exitcode, error(r.output)
1243\|  
1244\|          if message is not None:
1245\|              with (

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 32, in run_long_query
    result = node.query(
             ^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py", line 1242, in query
    assert r.exitcode == exitcode, error(r.output)
           ^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert r.exitcode == exitcode, error(r.output)

Description
  Received exception from server (version 25.8.14):
Code: 32. DB::Exception: Received from localhost:9000. DB::Exception: Attempt to read after eof: while receiving packet from clickhouse2:9000, 172.18.0.8, local address: 172.18.0.10:59382: While executing Remote. (ATTEMPT_TO_READ_AFTER_EOF)
(query: SELECT count(), hostName() 
            FROM datalakecatalog_db_0ea1b46f_fd49_11f0_a4fe_9200071779f7.`namespace_0ea1c2ea_fd49_11f0_8e6f_9200071779f7.table_0ea1c34b_fd49_11f0_8a68_9200071779f7` 
            WHERE NOT ignore(sleepEachRow(1)) 
            GROUP BY hostName()
            SETTINGS 
                object_storage_cluster='static_swarm_cluster', 
                max_threads=1


)

Assertion values
  assert r.exitcode == exitcode, error(r.output)
         ^ is 
  assert r.exitcode == exitcode, error(r.output)
         ^ is = 32
  assert r.exitcode == exitcode, error(r.output)
                       ^ is 138
  assert r.exitcode == exitcode, error(r.output)
                    ^ is = False
  assert r.exitcode == exitcode, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py', line 1242 in 'query'

1234\|                      **kwargs,
1235\|                  )
1236\|  
1237\|          if no_checks:
1238\|              return r
1239\|  
1240\|          if exitcode is not None:
1241\|              with Then(f"exitcode should be {exitcode}") if steps else NullStep():
1242\|>                 assert r.exitcode == exitcode, error(r.output)
1243\|  
1244\|          if message is not None:
1245\|              with (

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/regression.py", line 105, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/regression.py", line 98, in regression
    Feature(test=load("swarms.tests.feature", "feature"))(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/feature.py", line 17, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 343, in feature
    Scenario(test=swarm_out_of_disk_space)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 151, in swarm_out_of_disk_space
    actions.fill_clickhouse_disks(node=swarm_node)
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/steps/swarm_node_actions.py", line 97, in fill_clickhouse_disks
    clickhouse_limited_disk_config(node=node)
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/common.py", line 713, in add_config
    check_preprocessed_config_is_updated()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/common.py", line 637, in check_preprocessed_config_is_updated
    assert exitcode == 0, error()
           ^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert exitcode == 0, error()

Assertion values
  assert exitcode == 0, error()
         ^ is 1
  assert exitcode == 0, error()
                  ^ is = False
  assert exitcode == 0, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/common.py', line 637 in 'check_preprocessed_config_is_updated'

629\|          if settings.debug:
630\|              node.command(
631\|                  f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}"
632\|              )
633\|  
634\|          if after_removal:
635\|              assert exitcode == 1, error()
636\|          else:
637\|>             assert exitcode == 0, error()
638\|  
639\|      def wait_for_config_to_be_loaded(user=None):
640\|          """Wait for config to be loaded."""

AssertionError
Traceback (most recent call last):
  File "/usr/lib/python3.12/threading.py", line 1030, in _bootstrap
    self._bootstrap_inner()
  File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/node_failure.py", line 32, in run_long_query
    result = node.query(
             ^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py", line 1258, in query
    assert False, error(r.output)
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert False, error(r.output)

Description
  Received exception from server (version 25.8.14):
Code: 279. DB::Exception: Received from localhost:9000. DB::Exception: Cannot connect to any replica for query execution. (ALL_CONNECTION_TRIES_FAILED)
(query: SELECT count(), hostName() 
            FROM datalakecatalog_db_0ea1b46f_fd49_11f0_a4fe_9200071779f7.`namespace_0ea1c2ea_fd49_11f0_8e6f_9200071779f7.table_0ea1c34b_fd49_11f0_8a68_9200071779f7` 
            WHERE NOT ignore(sleepEachRow(0.5)) 
            GROUP BY hostName()
            SETTINGS 
                object_storage_cluster='static_swarm_cluster', 
                max_threads=1


)

Assertion values
  assert False, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/cluster.py', line 1258 in 'query'

1250\|                  assert message in r.output, error(r.output)
1251\|  
1252\|          if not ignore_exception:
1253\|              if message is None or "Exception:" not in message:
1254\|                  with Then("check if output has exception") if steps else NullStep():
1255\|                      if "Exception:" in r.output:
1256\|                          if raise_on_exception:
1257\|                              raise QueryRuntimeException(r.output)
1258\|>                         assert False, error(r.output)
1259\|  
1260\|          return r
1261\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/regression.py", line 105, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/regression.py", line 98, in regression
    Feature(test=load("swarms.tests.feature", "feature"))(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/feature.py", line 20, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/joins.py", line 679, in feature
    swarm_steps.add_node_to_swarm(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../swarms/tests/steps/swarm_steps.py", line 136, in add_node_to_swarm
    change_clickhouse_config(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/common.py", line 713, in add_config
    check_preprocessed_config_is_updated()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/common.py", line 637, in check_preprocessed_config_is_updated
    assert exitcode == 0, error()
           ^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert exitcode == 0, error()

Assertion values
  assert exitcode == 0, error()
         ^ is 1
  assert exitcode == 0, error()
                  ^ is = False
  assert exitcode == 0, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/swarms/../helpers/common.py', line 637 in 'check_preprocessed_config_is_updated'

629\|          if settings.debug:
630\|              node.command(
631\|                  f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}"
632\|              )
633\|  
634\|          if after_removal:
635\|              assert exitcode == 1, error()
636\|          else:
637\|>             assert exitcode == 0, error()
638\|  
639\|      def wait_for_config_to_be_loaded(user=None):
640\|          """Wait for config to be loaded."""