Iceberg Test Run Report

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 676, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 98, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 553, in feature
    Scenario(test=issue_with_decimal_column)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 459, in issue_with_decimal_column
    check_read_with_predicate_push_down(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 69, in check_read_with_predicate_push_down
    assert int(read_rows.output.strip()) == 0, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def check_read_with_predicate_push_down(
      self, where_clause, database_name, namespace, table_name
  ):
      """Check that ClickHouse does not read any rows when predicate pushdown is enabled
      and the condition does not match any data.
      """
      with By("drop all caches"):
          common.drop_all_caches(node=self.context.node)

      with And("read with input_format_parquet_filter_push_down"):
          log_comment_with_pruning = f"with_pruning_{getuid()}"
          result = iceberg_engine.read_data_from_clickhouse_iceberg_table(
              database_name=database_name,
              namespace=namespace,
              table_name=table_name,
              where_clause=where_clause,
              input_format_parquet_filter_push_down="1",
              log_comment=log_comment_with_pruning,
              use_iceberg_metadata_files_cache="0",
              use_iceberg_partition_pruning="0",
              input_format_parquet_bloom_filter_push_down="0",
          )
          assert result.output.strip() == "", error()

      with And(f"check that 0 rows were read"):
          read_rows = metrics.get_read_rows(
              log_comment=log_comment_with_pruning,
          )
          assert int(read_rows.output.strip()) == 0, error(

Description
  Expected 0 rows to be read, with where clause: decimal < 0.5, but got 10

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py', line 69 in 'check_read_with_predicate_push_down'

61\|              input_format_parquet_bloom_filter_push_down="0",
62\|          )
63\|          assert result.output.strip() == "", error()
64\|  
65\|      with And(f"check that 0 rows were read"):
66\|          read_rows = metrics.get_read_rows(
67\|              log_comment=log_comment_with_pruning,
68\|          )
69\|>         assert int(read_rows.output.strip()) == 0, error(
70\|              f"Expected 0 rows to be read, with where clause: {where_clause}, but got {read_rows.output.strip()}"
71\|          )
72\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 676, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 98, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 556, in feature
    Scenario(test=issue_with_float_column)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 529, in issue_with_float_column
    check_read_with_predicate_push_down(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 69, in check_read_with_predicate_push_down
    assert int(read_rows.output.strip()) == 0, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def check_read_with_predicate_push_down(
      self, where_clause, database_name, namespace, table_name
  ):
      """Check that ClickHouse does not read any rows when predicate pushdown is enabled
      and the condition does not match any data.
      """
      with By("drop all caches"):
          common.drop_all_caches(node=self.context.node)

      with And("read with input_format_parquet_filter_push_down"):
          log_comment_with_pruning = f"with_pruning_{getuid()}"
          result = iceberg_engine.read_data_from_clickhouse_iceberg_table(
              database_name=database_name,
              namespace=namespace,
              table_name=table_name,
              where_clause=where_clause,
              input_format_parquet_filter_push_down="1",
              log_comment=log_comment_with_pruning,
              use_iceberg_metadata_files_cache="0",
              use_iceberg_partition_pruning="0",
              input_format_parquet_bloom_filter_push_down="0",
          )
          assert result.output.strip() == "", error()

      with And(f"check that 0 rows were read"):
          read_rows = metrics.get_read_rows(
              log_comment=log_comment_with_pruning,
          )
          assert int(read_rows.output.strip()) == 0, error(

Description
  Expected 0 rows to be read, with where clause: float < 0.5, but got 10

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py', line 69 in 'check_read_with_predicate_push_down'

61\|              input_format_parquet_bloom_filter_push_down="0",
62\|          )
63\|          assert result.output.strip() == "", error()
64\|  
65\|      with And(f"check that 0 rows were read"):
66\|          read_rows = metrics.get_read_rows(
67\|              log_comment=log_comment_with_pruning,
68\|          )
69\|>         assert int(read_rows.output.strip()) == 0, error(
70\|              f"Expected 0 rows to be read, with where clause: {where_clause}, but got {read_rows.output.strip()}"
71\|          )
72\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 676, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 134, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 717, in feature
    Scenario(test=check_iceberg_partition_pruning_with_integer_type)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 161, in check_iceberg_partition_pruning_with_integer_type
    assert int(s3_read_requests_count) <= max_count, error()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert int(s3_read_requests_count) <= max_count, error()

Assertion values
  assert int(s3_read_requests_count) <= max_count, error()
             ^ is '74'
  assert int(s3_read_requests_count) <= max_count, error()
         ^ is = 74
  assert int(s3_read_requests_count) <= max_count, error()
                                        ^ is 53
  assert int(s3_read_requests_count) <= max_count, error()
                                     ^ is = False
  assert int(s3_read_requests_count) <= max_count, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py', line 161 in 'check_iceberg_partition_pruning_with_integer_type'

153\|                  # 2 requests for each row: one for metadata and one for data
154\|                  # 3-5 requests for metadata files
155\|                  s3_read_requests_count = metrics.get_S3ReadRequestsCount(
156\|                      log_comment=log_comment_with_partition_pruning,
157\|                  ).output.strip()
158\|                  min_count = 2 * (length - i) + 3
159\|                  max_count = 2 * (length - i) + 13
160\|                  assert min_count <= int(s3_read_requests_count), error()
161\|>                 assert int(s3_read_requests_count) <= max_count, error()
162\|  
163\|      with And("read data from ClickHouse with partition pruning disabled"):
164\|          for i in range(length):

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 676, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 134, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 720, in feature
    Scenario(test=check_partition_pruning_with_complex_where_clause)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 367, in check_partition_pruning_with_complex_where_clause
    assert int(s3_read_requests_count) <= 22, error()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert int(s3_read_requests_count) <= 22, error()

Assertion values
  assert int(s3_read_requests_count) <= 22, error()
             ^ is '28'
  assert int(s3_read_requests_count) <= 22, error()
         ^ is = 28
  assert int(s3_read_requests_count) <= 22, error()
                                     ^ is = False
  assert int(s3_read_requests_count) <= 22, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py', line 367 in 'check_partition_pruning_with_complex_where_clause'

359\|              assert int(min_max_pruned_files) == 43, error()
360\|  
361\|      with And("check that S3ReadRequestsCount is correct"):
362\|          s3_read_requests_count = metrics.get_S3ReadRequestsCount(
363\|              log_comment=log_comment_with_partition_pruning,
364\|          ).output.strip()
365\|          if check_clickhouse_version(">=25.4")(self) or check_if_antalya_build(self):
366\|              assert 10 <= int(s3_read_requests_count), error()
367\|>             assert int(s3_read_requests_count) <= 22, error()
368\|          else:
369\|              assert 95 <= int(s3_read_requests_count), error()
370\|              assert int(s3_read_requests_count) <= 100, error()

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 676, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 134, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 723, in feature
    Scenario(test=partition_pruning_with_date_type)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 512, in partition_pruning_with_date_type
    assert int(s3_read_requests_count) <= 150, error()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert int(s3_read_requests_count) <= 150, error()

Assertion values
  assert int(s3_read_requests_count) <= 150, error()
             ^ is '215'
  assert int(s3_read_requests_count) <= 150, error()
         ^ is = 215
  assert int(s3_read_requests_count) <= 150, error()
                                     ^ is = False
  assert int(s3_read_requests_count) <= 150, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py', line 512 in 'partition_pruning_with_date_type'

504\|              assert int(min_max_pruned_files) == 0, error()
505\|  
506\|      with And("check that S3ReadRequestsCount is correct"):
507\|          s3_read_requests_count = metrics.get_S3ReadRequestsCount(
508\|              log_comment=log_comment_with_partition_pruning,
509\|          ).output.strip()
510\|          if check_clickhouse_version(">=25.3")(self) or check_if_antalya_build(self):
511\|              assert 137 <= int(s3_read_requests_count), error()
512\|>             assert int(s3_read_requests_count) <= 150, error()
513\|          else:
514\|              assert 200 <= int(s3_read_requests_count), error()
515\|              assert int(s3_read_requests_count) <= 210, error()

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 680, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_table_engine/feature.py", line 33, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_table_engine/compaction.py", line 70, in feature
    Scenario(test=compaction_smoke)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_table_engine/compaction.py", line 54, in compaction_smoke
    iceberg_writes.optimize_iceberg_engine_table(table_name=ch_table_name)
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/steps/iceberg_writes.py", line 212, in optimize_iceberg_engine_table
    node.query(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py", line 1289, in query
    assert False, error(r.output)
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert False, error(r.output)

Description
  Received exception from server (version 26.6.2):
Code: 48. DB::Exception: Received from localhost:9000. DB::Exception: Unsupported snapshot's operation type OVERWRITE. (NOT_IMPLEMENTED)
(query: OPTIMIZE TABLE iceberg_compaction_0f34a3f9_a193_11f1_9ece_920009ae2786
)

Assertion values
  assert False, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py', line 1289 in 'query'

1281\|                  assert message in r.output, error(r.output)
1282\|  
1283\|          if not ignore_exception:
1284\|              if message is None or "Exception:" not in message:
1285\|                  with Then("check if output has exception") if steps else NullStep():
1286\|                      if "Exception:" in r.output:
1287\|                          if raise_on_exception:
1288\|                              raise QueryRuntimeException(r.output)
1289\|>                         assert False, error(r.output)
1290\|  
1291\|          return r
1292\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/partition_compatibility.py", line 524, in feature
    Scenario(test=reversed_destination_column_order_maps_by_name, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/partition_compatibility.py", line 452, in reversed_destination_column_order_maps_by_name
    export_partition(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_operations.py", line 304, in export_partition
    result = node.query(
             ^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py", line 1273, in query
    assert r.exitcode == exitcode, error(r.output)
           ^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert r.exitcode == exitcode, error(r.output)

Description
  Received exception from server (version 26.6.2):
Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Cannot export to default.iceberg_3d892dfb_a193_11f1_8931_920009ae2786: partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'. EXPORT PART/PARTITION matches columns by position, so partition key columns must be declared at the same position in both tables.. (BAD_ARGUMENTS)
(query: ALTER TABLE mt_3d4b70c9_a193_11f1_9aa9_920009ae2786 EXPORT PARTITION ID '1' TO TABLE iceberg_3d892dfb_a193_11f1_8931_920009ae2786
)

Assertion values
  assert r.exitcode == exitcode, error(r.output)
         ^ is 
  assert r.exitcode == exitcode, error(r.output)
         ^ is = 36
  assert r.exitcode == exitcode, error(r.output)
                       ^ is 0
  assert r.exitcode == exitcode, error(r.output)
                    ^ is = False
  assert r.exitcode == exitcode, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py', line 1273 in 'query'

1265\|                      **kwargs,
1266\|                  )
1267\|  
1268\|          if no_checks:
1269\|              return r
1270\|  
1271\|          if exitcode is not None:
1272\|              with Then(f"exitcode should be {exitcode}") if steps else NullStep():
1273\|>                 assert r.exitcode == exitcode, error(r.output)
1274\|  
1275\|          if message is not None:
1276\|              with (

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 302, in value_counts_sum_to_row_count
    assert_value_counts_sum_to(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 568, in assert_value_counts_sum_to
    assert not bad, error(
           ^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_value_counts_sum_to(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_total,
  ):
      """For each column, assert ``sum(data_file.value_counts[field_id]) == expected_total``.

      This cross-checks the per-file statistics against the total row count
      without needing to decode ``lower_bounds`` / ``upper_bounds`` bytes.
      """
      table = load_pyiceberg_table(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      schema = table.schema()

      per_column_totals = {}
      for data_file in data_files:
          for field_id, count in (data_file.value_counts or {}).items():
              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count

      bad = []
      for field in schema.fields:
          total = per_column_totals.get(field.field_id, 0)
          if total != expected_total:
              bad.append((field.name, field.field_id, total))

      assert not bad, error(

Description
  Value_counts do not sum to 7 for iceberg_71af81c0_a193_11f1_a151_920009ae2786: [('id', 1, 0), ('year', 2, 0)]

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 568 in 'assert_value_counts_sum_to'

560\|              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count
561\|  
562\|      bad = []
563\|      for field in schema.fields:
564\|          total = per_column_totals.get(field.field_id, 0)
565\|          if total != expected_total:
566\|              bad.append((field.name, field.field_id, total))
567\|  
568\|>     assert not bad, error(
569\|          f"value_counts do not sum to {expected_total} for "
570\|          f"{as_destination_name(destination)}: {bad!r}"
571\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 357, in data_file_paths_under_table_prefix
    assert_file_paths_under_prefix(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 597, in assert_file_paths_under_prefix
    assert not bad_paths, error(
           ^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_file_paths_under_prefix(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_prefix,
  ):
      """Assert that every data file's path begins with ``expected_prefix``.

      Useful when validating ``write_full_path_in_iceberg_metadata`` and mixed
      storage paths.
      """
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      bad_paths = [
          df.file_path
          for df in data_files
          if not df.file_path.startswith(expected_prefix)
      ]
      assert not bad_paths, error(

Description
  Data files not under expected prefix 's3://warehouse/data/iceberg_72338249_a193_11f1_8cae_920009ae2786/' in iceberg_72338249_a193_11f1_8cae_920009ae2786: ['data/iceberg_72338249_a193_11f1_8cae_920009ae2786/data/data-22b51872-a2d6-4601-b0f4-40bc84c3e26a.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 597 in 'assert_file_paths_under_prefix'

589\|          minio_root_user=minio_root_user,
590\|          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
591\|      )
592\|      bad_paths = [
593\|          df.file_path
594\|          for df in data_files
595\|          if not df.file_path.startswith(expected_prefix)
596\|      ]
597\|>     assert not bad_paths, error(
598\|          f"Data files not under expected prefix {expected_prefix!r} in "
599\|          f"{as_destination_name(destination)}: {bad_paths!r}"
600\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 424, in external_reader_round_trips_exported_data
    arrow_table = table.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1367, in _task_to_record_batches
    with fs.open_input_file(path) as fin:
         ^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_fs.pyx", line 787, in pyarrow._fs.FileSystem.open_input_file
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_72b5fbcc_a193_11f1_8f98_920009ae2786/data/data-5f568735-4826-44d2-982f-bfabae47418a.parquet'. Detail: [errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 446, in external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_ManifestIntegrity_ExternalReader("1.0"))
  @Name("external iceberg reader round-trips exported data")
  def external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg (a non-ClickHouse reader) can scan the destination and
      materialise the exported rows.

      Currently XFail (https://github.com/Altinity/ClickHouse/issues/2161):
      EXPORT PARTITION writes Parquet without Iceberg field-ids and IcebergS3
      tables do not set ``schema.name-mapping.default``, so strict readers
      (PyIceberg) cannot map physical columns to the table schema. On older
      builds that still write bucket-relative ``data_file.file_path`` URIs, the
      same scenario may instead fail at FileIO open with ``FileNotFoundError``.
      """
      source_table = f"mt_{getuid()}"
      expected_values = [(1, 2020), (2, 2020), (3, 2020)]
      values_sql = ", ".join(f"({id_}, {year})" for id_, year in expected_values)

      with Given("create source table and insert a known partition"):
          create_replicated_mergetree(
              table_name=source_table,
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
          )
          insert_data(table_name=source_table, values=values_sql)

      with And("create the Iceberg destination with full paths in metadata"):
          # write_full_path_in_iceberg_metadata = 1 is required infrastructure:
          # without it the manifest-list pointer in metadata.json is also
          # bucket-relative and PyIceberg fails before we ever reach a
          # data_file.file_path. We want the failure to be about the data
          # files, not the manifest list.
          destination = create_iceberg_destination(
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              query_settings=FULL_PATHS_SETTING,
          )

      with When("export the partition with full paths in metadata"):
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with And("load the destination via PyIceberg (external reader)"):
          table = load_pyiceberg_table(
              destination=destination,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with Then("PyIceberg can scan the table and materialise the rows"):
          try:
              arrow_table = table.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(
                  "External reader (PyIceberg) could not map exported Parquet "
                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
                  "Parquet without Iceberg field-ids, and IcebergS3 "
                  "destinations do not set table property "
                  "schema.name-mapping.default. Underlying error: "
                  f"{type(exc).__name__}: {exc}"
              )
          except (FileNotFoundError, OSError) as exc:
              offending = [
                  df.file_path
                  for df in get_data_files(
                      destination=destination,
                      minio_root_user=minio_root_user,
                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  )
              ]
              assert False, error(

Description
  External reader (PyIceberg) failed to open a data file written by EXPORT PARTITION. This is the user-visible effect of data_file.file_path being bucket-relative instead of an absolute URI — FileIO dispatch falls back to the local filesystem. Underlying error: FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_72b5fbcc_a193_11f1_8f98_920009ae2786/data/data-5f568735-4826-44d2-982f-bfabae47418a.parquet'. Detail: [errno 2] No such file or directory. data_file.file_path values in this snapshot: ['data/iceberg_72b5fbcc_a193_11f1_8f98_920009ae2786/data/data-5f568735-4826-44d2-982f-bfabae47418a.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py', line 446 in 'external_reader_round_trips_exported_data'

438\|              offending = [
439\|                  df.file_path
440\|                  for df in get_data_files(
441\|                      destination=destination,
442\|                      minio_root_user=minio_root_user,
443\|                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
444\|                  )
445\|              ]
446\|>             assert False, error(
447\|                  f"External reader (PyIceberg) failed to open a data file "
448\|                  f"written by EXPORT PARTITION. This is the user-visible "
449\|                  f"effect of data_file.file_path being bucket-relative "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 561, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 454, in no_catalog_drop_with_purge_recreates_same_table
    node.query(create_sql, settings=FULL_PATHS_SETTING)
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py", line 1289, in query
    assert False, error(r.output)
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert False, error(r.output)

Description
  Received exception from server (version 26.6.2):
Code: 57. DB::Exception: Received from localhost:9000. DB::Exception: Iceberg table with path data/iceberg_746e75d3_a193_11f1_9696_920009ae2786/ already exists. (TABLE_ALREADY_EXISTS)
(query: CREATE TABLE iceberg_746e75d3_a193_11f1_9696_920009ae2786 (id Int64, year Int32)
ENGINE = IcebergS3('http://minio:9000/warehouse/data/iceberg_746e75d3_a193_11f1_9696_920009ae2786/', '[masked]:Secret(name='minio_root_user')', '[masked]:Secret(name='minio_root_password')')
PARTITION BY year
SETTINGS s3_retry_attempts = 1
)

Assertion values
  assert False, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py', line 1289 in 'query'

1281\|                  assert message in r.output, error(r.output)
1282\|  
1283\|          if not ignore_exception:
1284\|              if message is None or "Exception:" not in message:
1285\|                  with Then("check if output has exception") if steps else NullStep():
1286\|                      if "Exception:" in r.output:
1287\|                          if raise_on_exception:
1288\|                              raise QueryRuntimeException(r.output)
1289\|>                         assert False, error(r.output)
1290\|  
1291\|          return r
1292\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/transactions.py", line 658, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/transactions.py", line 355, in commit_survives_pre_publish_failure
    export_partition(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_operations.py", line 314, in export_partition
    wait_for_export_status(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_status.py", line 218, in wait_for_export_status
    assert last_status == expected_status, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(When)
  def wait_for_export_status(
      self,
      source_table,
      partition_id,
      expected_status="COMPLETED",
      destination=None,
      destination_table=None,
      timeout=120,
      delay=3,
      node=None,
  ):
      """Poll the export status system table until ``status`` matches.

      Fails the scenario on timeout, reporting the last observed status.

      See :func:`get_export_row` for the meaning of ``destination`` /
      ``destination_table``.
      """
      if node is None:
          node = self.context.node

      last_status = None
      for attempt in retries(timeout=timeout, delay=delay):
          with attempt:
              last_status = get_export_row(
                  source_table=source_table,
                  partition_id=partition_id,
                  destination=destination,
                  destination_table=destination_table,
                  columns="status",
                  node=node,
              )
              assert last_status == expected_status, error(

Description
  Expected status 'COMPLETED' for source_table='mt_7adf847c_a193_11f1_890f_920009ae2786' partition_id='2020' but got 'FAILED'

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_status.py', line 218 in 'wait_for_export_status'

210\|              last_status = get_export_row(
211\|                  source_table=source_table,
212\|                  partition_id=partition_id,
213\|                  destination=destination,
214\|                  destination_table=destination_table,
215\|                  columns="status",
216\|                  node=node,
217\|              )
218\|>             assert last_status == expected_status, error(
219\|                  f"Expected status '{expected_status}' for "
220\|                  f"source_table='{source_table}' partition_id='{partition_id}'"
221\|                  f" but got '{last_status}'"

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 302, in value_counts_sum_to_row_count
    assert_value_counts_sum_to(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 568, in assert_value_counts_sum_to
    assert not bad, error(
           ^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_value_counts_sum_to(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_total,
  ):
      """For each column, assert ``sum(data_file.value_counts[field_id]) == expected_total``.

      This cross-checks the per-file statistics against the total row count
      without needing to decode ``lower_bounds`` / ``upper_bounds`` bytes.
      """
      table = load_pyiceberg_table(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      schema = table.schema()

      per_column_totals = {}
      for data_file in data_files:
          for field_id, count in (data_file.value_counts or {}).items():
              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count

      bad = []
      for field in schema.fields:
          total = per_column_totals.get(field.field_id, 0)
          if total != expected_total:
              bad.append((field.name, field.field_id, total))

      assert not bad, error(

Description
  Value_counts do not sum to 7 for datalake_63fe271b_a194_11f1_a83e_920009ae2786.\`export_ns_63fe265f_a194_11f1_8e77_920009ae2786.iceberg_63fe26e2_a194_11f1_9613_920009ae2786\`: [('id', 1, 0), ('year', 2, 0)]

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 568 in 'assert_value_counts_sum_to'

560\|              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count
561\|  
562\|      bad = []
563\|      for field in schema.fields:
564\|          total = per_column_totals.get(field.field_id, 0)
565\|          if total != expected_total:
566\|              bad.append((field.name, field.field_id, total))
567\|  
568\|>     assert not bad, error(
569\|          f"value_counts do not sum to {expected_total} for "
570\|          f"{as_destination_name(destination)}: {bad!r}"
571\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 357, in data_file_paths_under_table_prefix
    assert_file_paths_under_prefix(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 597, in assert_file_paths_under_prefix
    assert not bad_paths, error(
           ^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_file_paths_under_prefix(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_prefix,
  ):
      """Assert that every data file's path begins with ``expected_prefix``.

      Useful when validating ``write_full_path_in_iceberg_metadata`` and mixed
      storage paths.
      """
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      bad_paths = [
          df.file_path
          for df in data_files
          if not df.file_path.startswith(expected_prefix)
      ]
      assert not bad_paths, error(

Description
  Data files not under expected prefix 's3://warehouse/data/iceberg_671d2777_a194_11f1_b5c0_920009ae2786/' in datalake_671d27bc_a194_11f1_9bfb_920009ae2786.\`export_ns_671d269f_a194_11f1_8925_920009ae2786.iceberg_671d2777_a194_11f1_b5c0_920009ae2786\`: ['data/iceberg_671d2777_a194_11f1_b5c0_920009ae2786/data/data-3a4376fc-686a-4895-8e1f-128232de56c8.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 597 in 'assert_file_paths_under_prefix'

589\|          minio_root_user=minio_root_user,
590\|          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
591\|      )
592\|      bad_paths = [
593\|          df.file_path
594\|          for df in data_files
595\|          if not df.file_path.startswith(expected_prefix)
596\|      ]
597\|>     assert not bad_paths, error(
598\|          f"Data files not under expected prefix {expected_prefix!r} in "
599\|          f"{as_destination_name(destination)}: {bad_paths!r}"
600\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 424, in external_reader_round_trips_exported_data
    arrow_table = table.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1367, in _task_to_record_batches
    with fs.open_input_file(path) as fin:
         ^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_fs.pyx", line 787, in pyarrow._fs.FileSystem.open_input_file
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_6a35a53a_a194_11f1_a2d2_920009ae2786/data/data-454f714c-b269-4f07-a4b6-ab7eb2099be7.parquet'. Detail: [errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 446, in external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_ManifestIntegrity_ExternalReader("1.0"))
  @Name("external iceberg reader round-trips exported data")
  def external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg (a non-ClickHouse reader) can scan the destination and
      materialise the exported rows.

      Currently XFail (https://github.com/Altinity/ClickHouse/issues/2161):
      EXPORT PARTITION writes Parquet without Iceberg field-ids and IcebergS3
      tables do not set ``schema.name-mapping.default``, so strict readers
      (PyIceberg) cannot map physical columns to the table schema. On older
      builds that still write bucket-relative ``data_file.file_path`` URIs, the
      same scenario may instead fail at FileIO open with ``FileNotFoundError``.
      """
      source_table = f"mt_{getuid()}"
      expected_values = [(1, 2020), (2, 2020), (3, 2020)]
      values_sql = ", ".join(f"({id_}, {year})" for id_, year in expected_values)

      with Given("create source table and insert a known partition"):
          create_replicated_mergetree(
              table_name=source_table,
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
          )
          insert_data(table_name=source_table, values=values_sql)

      with And("create the Iceberg destination with full paths in metadata"):
          # write_full_path_in_iceberg_metadata = 1 is required infrastructure:
          # without it the manifest-list pointer in metadata.json is also
          # bucket-relative and PyIceberg fails before we ever reach a
          # data_file.file_path. We want the failure to be about the data
          # files, not the manifest list.
          destination = create_iceberg_destination(
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              query_settings=FULL_PATHS_SETTING,
          )

      with When("export the partition with full paths in metadata"):
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with And("load the destination via PyIceberg (external reader)"):
          table = load_pyiceberg_table(
              destination=destination,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with Then("PyIceberg can scan the table and materialise the rows"):
          try:
              arrow_table = table.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(
                  "External reader (PyIceberg) could not map exported Parquet "
                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
                  "Parquet without Iceberg field-ids, and IcebergS3 "
                  "destinations do not set table property "
                  "schema.name-mapping.default. Underlying error: "
                  f"{type(exc).__name__}: {exc}"
              )
          except (FileNotFoundError, OSError) as exc:
              offending = [
                  df.file_path
                  for df in get_data_files(
                      destination=destination,
                      minio_root_user=minio_root_user,
                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  )
              ]
              assert False, error(

Description
  External reader (PyIceberg) failed to open a data file written by EXPORT PARTITION. This is the user-visible effect of data_file.file_path being bucket-relative instead of an absolute URI — FileIO dispatch falls back to the local filesystem. Underlying error: FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_6a35a53a_a194_11f1_a2d2_920009ae2786/data/data-454f714c-b269-4f07-a4b6-ab7eb2099be7.parquet'. Detail: [errno 2] No such file or directory. data_file.file_path values in this snapshot: ['data/iceberg_6a35a53a_a194_11f1_a2d2_920009ae2786/data/data-454f714c-b269-4f07-a4b6-ab7eb2099be7.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py', line 446 in 'external_reader_round_trips_exported_data'

438\|              offending = [
439\|                  df.file_path
440\|                  for df in get_data_files(
441\|                      destination=destination,
442\|                      minio_root_user=minio_root_user,
443\|                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
444\|                  )
445\|              ]
446\|>             assert False, error(
447\|                  f"External reader (PyIceberg) failed to open a data file "
448\|                  f"written by EXPORT PARTITION. This is the user-visible "
449\|                  f"effect of data_file.file_path being bucket-relative "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 343, in catalog_external_reader_round_trips_exported_data
    arrow_table = external_view.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1367, in _task_to_record_batches
    with fs.open_input_file(path) as fin:
         ^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_fs.pyx", line 787, in pyarrow._fs.FileSystem.open_input_file
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_7085b043_a194_11f1_b827_920009ae2786/data/data-ea3a20cc-52f3-45af-8c56-da66faa83559.parquet'. Detail: [errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 561, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 362, in catalog_external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_CatalogIntegration_RestGlue("1.0"))
  @Name("catalog: external reader round-trips exported data")
  def catalog_external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg, following the catalog's metadata pointer, reads back
      the exported rows byte-exact. Catalog-mode analogue of the
      ``manifest_integrity`` external-reader scenario; skipped under
      ``no_catalog``.

      Currently XFail: https://github.com/Altinity/ClickHouse/issues/2161
      """
      source_table = _seed_source()

      with Given("materialise a catalog-backed Iceberg destination via PyIceberg"):
          destination = create_pyiceberg_catalog_destination(
              schema=CATALOG_SCHEMA,
              partition_spec=CATALOG_PARTITION_SPEC,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with When("export the single partition"):
          # ``destination=`` dispatches the catalog-aware split of
          # (destination_database, destination_table) when waiting for the
          # status to flip to ``COMPLETED``; see Phase 2 notes in
          # ``steps/export_status.py``.
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with Then("PyIceberg (external reader) can materialise the exported rows"):
          handle = as_pyiceberg_handle(destination)
          assert handle is not None, error(
              "Expected catalog-backed destination to expose a PyIceberg handle"
          )
          external_view = handle["pyiceberg_catalog"].load_table(
              f"{handle['namespace']}.{handle['table_name']}"
          )
          try:
              arrow_table = external_view.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(
                  "External reader (PyIceberg) could not map exported Parquet "
                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
                  "Parquet without Iceberg field-ids even when the catalog "
                  "table was created with a proper schema. Underlying error: "
                  f"{type(exc).__name__}: {exc}"
              )
          except (FileNotFoundError, OSError) as exc:
              offending = []
              current_snapshot = external_view.current_snapshot()
              if current_snapshot is not None:
                  for manifest in current_snapshot.manifests(external_view.io):
                      for entry in manifest.fetch_manifest_entry(external_view.io):
                          offending.append(entry.data_file.file_path)
              assert False, error(

Description
  External reader (PyIceberg) could not open a data file committed through the catalog. FileIO dispatch fell back to the local filesystem, which is the symptom of data_file.file_path lacking a URI scheme. Underlying error: FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_7085b043_a194_11f1_b827_920009ae2786/data/data-ea3a20cc-52f3-45af-8c56-da66faa83559.parquet'. Detail: [errno 2] No such file or directory. data_file.file_path values: ['data/iceberg_7085b043_a194_11f1_b827_920009ae2786/data/data-ea3a20cc-52f3-45af-8c56-da66faa83559.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py', line 362 in 'catalog_external_reader_round_trips_exported_data'

354\|              )
355\|          except (FileNotFoundError, OSError) as exc:
356\|              offending = []
357\|              current_snapshot = external_view.current_snapshot()
358\|              if current_snapshot is not None:
359\|                  for manifest in current_snapshot.manifests(external_view.io):
360\|                      for entry in manifest.fetch_manifest_entry(external_view.io):
361\|                          offending.append(entry.data_file.file_path)
362\|>             assert False, error(
363\|                  f"External reader (PyIceberg) could not open a data file "
364\|                  f"committed through the catalog. FileIO dispatch fell back "
365\|                  f"to the local filesystem, which is the symptom of "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 561, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 511, in catalog_drop_with_purge_recreates_same_table
    node.query(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py", line 1289, in query
    assert False, error(r.output)
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert False, error(r.output)

Description
  Received exception from server (version 26.6.2):
Code: 57. DB::Exception: Received from localhost:9000. DB::Exception: Iceberg table with path data/iceberg_737b7a87_a194_11f1_9997_920009ae2786/ already exists. (TABLE_ALREADY_EXISTS)
(query: CREATE TABLE datalake_737b7ab9_a194_11f1_b17d_920009ae2786.`export_ns_737b7a3c_a194_11f1_888b_920009ae2786.iceberg_737b7a87_a194_11f1_9997_920009ae2786` (id Int64, year Int32)
ENGINE = IcebergS3('http://minio:9000/warehouse/data/iceberg_737b7a87_a194_11f1_9997_920009ae2786/', '[masked]:Secret(name='minio_root_user')', '[masked]:Secret(name='minio_root_password')')
PARTITION BY year
SETTINGS s3_retry_attempts = 1, iceberg_format_version = 2)

Assertion values
  assert False, error(r.output)
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/cluster.py', line 1289 in 'query'

1281\|                  assert message in r.output, error(r.output)
1282\|  
1283\|          if not ignore_exception:
1284\|              if message is None or "Exception:" not in message:
1285\|                  with Then("check if output has exception") if steps else NullStep():
1286\|                      if "Exception:" in r.output:
1287\|                          if raise_on_exception:
1288\|                              raise QueryRuntimeException(r.output)
1289\|>                         assert False, error(r.output)
1290\|  
1291\|          return r
1292\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/transactions.py", line 658, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/transactions.py", line 355, in commit_survives_pre_publish_failure
    export_partition(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_operations.py", line 314, in export_partition
    wait_for_export_status(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_status.py", line 218, in wait_for_export_status
    assert last_status == expected_status, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(When)
  def wait_for_export_status(
      self,
      source_table,
      partition_id,
      expected_status="COMPLETED",
      destination=None,
      destination_table=None,
      timeout=120,
      delay=3,
      node=None,
  ):
      """Poll the export status system table until ``status`` matches.

      Fails the scenario on timeout, reporting the last observed status.

      See :func:`get_export_row` for the meaning of ``destination`` /
      ``destination_table``.
      """
      if node is None:
          node = self.context.node

      last_status = None
      for attempt in retries(timeout=timeout, delay=delay):
          with attempt:
              last_status = get_export_row(
                  source_table=source_table,
                  partition_id=partition_id,
                  destination=destination,
                  destination_table=destination_table,
                  columns="status",
                  node=node,
              )
              assert last_status == expected_status, error(

Description
  Expected status 'COMPLETED' for source_table='mt_886bd9a0_a194_11f1_b2ea_920009ae2786' partition_id='2020' but got 'FAILED'

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_status.py', line 218 in 'wait_for_export_status'

210\|              last_status = get_export_row(
211\|                  source_table=source_table,
212\|                  partition_id=partition_id,
213\|                  destination=destination,
214\|                  destination_table=destination_table,
215\|                  columns="status",
216\|                  node=node,
217\|              )
218\|>             assert last_status == expected_status, error(
219\|                  f"Expected status '{expected_status}' for "
220\|                  f"source_table='{source_table}' partition_id='{partition_id}'"
221\|                  f" but got '{last_status}'"

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 302, in value_counts_sum_to_row_count
    assert_value_counts_sum_to(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 568, in assert_value_counts_sum_to
    assert not bad, error(
           ^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_value_counts_sum_to(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_total,
  ):
      """For each column, assert ``sum(data_file.value_counts[field_id]) == expected_total``.

      This cross-checks the per-file statistics against the total row count
      without needing to decode ``lower_bounds`` / ``upper_bounds`` bytes.
      """
      table = load_pyiceberg_table(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      schema = table.schema()

      per_column_totals = {}
      for data_file in data_files:
          for field_id, count in (data_file.value_counts or {}).items():
              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count

      bad = []
      for field in schema.fields:
          total = per_column_totals.get(field.field_id, 0)
          if total != expected_total:
              bad.append((field.name, field.field_id, total))

      assert not bad, error(

Description
  Value_counts do not sum to 7 for datalake_7b30427b_a196_11f1_9910_920009ae2786.\`export_ns_7b30410c_a196_11f1_ad3f_920009ae2786.iceberg_7b304238_a196_11f1_a612_920009ae2786\`: [('id', 1, 0), ('year', 2, 0)]

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 568 in 'assert_value_counts_sum_to'

560\|              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count
561\|  
562\|      bad = []
563\|      for field in schema.fields:
564\|          total = per_column_totals.get(field.field_id, 0)
565\|          if total != expected_total:
566\|              bad.append((field.name, field.field_id, total))
567\|  
568\|>     assert not bad, error(
569\|          f"value_counts do not sum to {expected_total} for "
570\|          f"{as_destination_name(destination)}: {bad!r}"
571\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 357, in data_file_paths_under_table_prefix
    assert_file_paths_under_prefix(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 597, in assert_file_paths_under_prefix
    assert not bad_paths, error(
           ^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_file_paths_under_prefix(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_prefix,
  ):
      """Assert that every data file's path begins with ``expected_prefix``.

      Useful when validating ``write_full_path_in_iceberg_metadata`` and mixed
      storage paths.
      """
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      bad_paths = [
          df.file_path
          for df in data_files
          if not df.file_path.startswith(expected_prefix)
      ]
      assert not bad_paths, error(

Description
  Data files not under expected prefix 's3://warehouse/data/iceberg_7c83afd6_a196_11f1_a548_920009ae2786/' in datalake_7c83b010_a196_11f1_bd64_920009ae2786.\`export_ns_7c83af21_a196_11f1_b035_920009ae2786.iceberg_7c83afd6_a196_11f1_a548_920009ae2786\`: ['data/iceberg_7c83afd6_a196_11f1_a548_920009ae2786/data/data-b0632404-0c9b-4005-a491-1bdd02905790.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 597 in 'assert_file_paths_under_prefix'

589\|          minio_root_user=minio_root_user,
590\|          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
591\|      )
592\|      bad_paths = [
593\|          df.file_path
594\|          for df in data_files
595\|          if not df.file_path.startswith(expected_prefix)
596\|      ]
597\|>     assert not bad_paths, error(
598\|          f"Data files not under expected prefix {expected_prefix!r} in "
599\|          f"{as_destination_name(destination)}: {bad_paths!r}"
600\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 424, in external_reader_round_trips_exported_data
    arrow_table = table.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1367, in _task_to_record_batches
    with fs.open_input_file(path) as fin:
         ^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_fs.pyx", line 787, in pyarrow._fs.FileSystem.open_input_file
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_7de10a2b_a196_11f1_8ff1_920009ae2786/data/data-f156504d-20e0-4044-98d0-0f4252056ee7.parquet'. Detail: [errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 481, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 446, in external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_ManifestIntegrity_ExternalReader("1.0"))
  @Name("external iceberg reader round-trips exported data")
  def external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg (a non-ClickHouse reader) can scan the destination and
      materialise the exported rows.

      Currently XFail (https://github.com/Altinity/ClickHouse/issues/2161):
      EXPORT PARTITION writes Parquet without Iceberg field-ids and IcebergS3
      tables do not set ``schema.name-mapping.default``, so strict readers
      (PyIceberg) cannot map physical columns to the table schema. On older
      builds that still write bucket-relative ``data_file.file_path`` URIs, the
      same scenario may instead fail at FileIO open with ``FileNotFoundError``.
      """
      source_table = f"mt_{getuid()}"
      expected_values = [(1, 2020), (2, 2020), (3, 2020)]
      values_sql = ", ".join(f"({id_}, {year})" for id_, year in expected_values)

      with Given("create source table and insert a known partition"):
          create_replicated_mergetree(
              table_name=source_table,
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
          )
          insert_data(table_name=source_table, values=values_sql)

      with And("create the Iceberg destination with full paths in metadata"):
          # write_full_path_in_iceberg_metadata = 1 is required infrastructure:
          # without it the manifest-list pointer in metadata.json is also
          # bucket-relative and PyIceberg fails before we ever reach a
          # data_file.file_path. We want the failure to be about the data
          # files, not the manifest list.
          destination = create_iceberg_destination(
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              query_settings=FULL_PATHS_SETTING,
          )

      with When("export the partition with full paths in metadata"):
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with And("load the destination via PyIceberg (external reader)"):
          table = load_pyiceberg_table(
              destination=destination,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with Then("PyIceberg can scan the table and materialise the rows"):
          try:
              arrow_table = table.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(
                  "External reader (PyIceberg) could not map exported Parquet "
                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
                  "Parquet without Iceberg field-ids, and IcebergS3 "
                  "destinations do not set table property "
                  "schema.name-mapping.default. Underlying error: "
                  f"{type(exc).__name__}: {exc}"
              )
          except (FileNotFoundError, OSError) as exc:
              offending = [
                  df.file_path
                  for df in get_data_files(
                      destination=destination,
                      minio_root_user=minio_root_user,
                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  )
              ]
              assert False, error(

Description
  External reader (PyIceberg) failed to open a data file written by EXPORT PARTITION. This is the user-visible effect of data_file.file_path being bucket-relative instead of an absolute URI — FileIO dispatch falls back to the local filesystem. Underlying error: FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_7de10a2b_a196_11f1_8ff1_920009ae2786/data/data-f156504d-20e0-4044-98d0-0f4252056ee7.parquet'. Detail: [errno 2] No such file or directory. data_file.file_path values in this snapshot: ['data/iceberg_7de10a2b_a196_11f1_8ff1_920009ae2786/data/data-f156504d-20e0-4044-98d0-0f4252056ee7.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py', line 446 in 'external_reader_round_trips_exported_data'

438\|              offending = [
439\|                  df.file_path
440\|                  for df in get_data_files(
441\|                      destination=destination,
442\|                      minio_root_user=minio_root_user,
443\|                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
444\|                  )
445\|              ]
446\|>             assert False, error(
447\|                  f"External reader (PyIceberg) failed to open a data file "
448\|                  f"written by EXPORT PARTITION. This is the user-visible "
449\|                  f"effect of data_file.file_path being bucket-relative "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 343, in catalog_external_reader_round_trips_exported_data
    arrow_table = external_view.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1367, in _task_to_record_batches
    with fs.open_input_file(path) as fin:
         ^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_fs.pyx", line 787, in pyarrow._fs.FileSystem.open_input_file
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_82d265ca_a196_11f1_980b_920009ae2786/data/data-28ff64da-11b0-4dd7-b82f-d50c0aa97591.parquet'. Detail: [errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 561, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 362, in catalog_external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_CatalogIntegration_RestGlue("1.0"))
  @Name("catalog: external reader round-trips exported data")
  def catalog_external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg, following the catalog's metadata pointer, reads back
      the exported rows byte-exact. Catalog-mode analogue of the
      ``manifest_integrity`` external-reader scenario; skipped under
      ``no_catalog``.

      Currently XFail: https://github.com/Altinity/ClickHouse/issues/2161
      """
      source_table = _seed_source()

      with Given("materialise a catalog-backed Iceberg destination via PyIceberg"):
          destination = create_pyiceberg_catalog_destination(
              schema=CATALOG_SCHEMA,
              partition_spec=CATALOG_PARTITION_SPEC,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with When("export the single partition"):
          # ``destination=`` dispatches the catalog-aware split of
          # (destination_database, destination_table) when waiting for the
          # status to flip to ``COMPLETED``; see Phase 2 notes in
          # ``steps/export_status.py``.
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with Then("PyIceberg (external reader) can materialise the exported rows"):
          handle = as_pyiceberg_handle(destination)
          assert handle is not None, error(
              "Expected catalog-backed destination to expose a PyIceberg handle"
          )
          external_view = handle["pyiceberg_catalog"].load_table(
              f"{handle['namespace']}.{handle['table_name']}"
          )
          try:
              arrow_table = external_view.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(
                  "External reader (PyIceberg) could not map exported Parquet "
                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
                  "Parquet without Iceberg field-ids even when the catalog "
                  "table was created with a proper schema. Underlying error: "
                  f"{type(exc).__name__}: {exc}"
              )
          except (FileNotFoundError, OSError) as exc:
              offending = []
              current_snapshot = external_view.current_snapshot()
              if current_snapshot is not None:
                  for manifest in current_snapshot.manifests(external_view.io):
                      for entry in manifest.fetch_manifest_entry(external_view.io):
                          offending.append(entry.data_file.file_path)
              assert False, error(

Description
  External reader (PyIceberg) could not open a data file committed through the catalog. FileIO dispatch fell back to the local filesystem, which is the symptom of data_file.file_path lacking a URI scheme. Underlying error: FileNotFoundError: [Errno 2] Failed to open local file '/home/ubuntu/_work/ClickHouse/ClickHouse/data/iceberg_82d265ca_a196_11f1_980b_920009ae2786/data/data-28ff64da-11b0-4dd7-b82f-d50c0aa97591.parquet'. Detail: [errno 2] No such file or directory. data_file.file_path values: ['data/iceberg_82d265ca_a196_11f1_980b_920009ae2786/data/data-28ff64da-11b0-4dd7-b82f-d50c0aa97591.parquet']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py', line 362 in 'catalog_external_reader_round_trips_exported_data'

354\|              )
355\|          except (FileNotFoundError, OSError) as exc:
356\|              offending = []
357\|              current_snapshot = external_view.current_snapshot()
358\|              if current_snapshot is not None:
359\|                  for manifest in current_snapshot.manifests(external_view.io):
360\|                      for entry in manifest.fetch_manifest_entry(external_view.io):
361\|                          offending.append(entry.data_file.file_path)
362\|>             assert False, error(
363\|                  f"External reader (PyIceberg) could not open a data file "
364\|                  f"committed through the catalog. FileIO dispatch fell back "
365\|                  f"to the local filesystem, which is the symptom of "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 714, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 180, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 451, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 696, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 118, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 70, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/transactions.py", line 658, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/transactions.py", line 355, in commit_survives_pre_publish_failure
    export_partition(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_operations.py", line 314, in export_partition
    wait_for_export_status(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_status.py", line 218, in wait_for_export_status
    assert last_status == expected_status, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(When)
  def wait_for_export_status(
      self,
      source_table,
      partition_id,
      expected_status="COMPLETED",
      destination=None,
      destination_table=None,
      timeout=120,
      delay=3,
      node=None,
  ):
      """Poll the export status system table until ``status`` matches.

      Fails the scenario on timeout, reporting the last observed status.

      See :func:`get_export_row` for the meaning of ``destination`` /
      ``destination_table``.
      """
      if node is None:
          node = self.context.node

      last_status = None
      for attempt in retries(timeout=timeout, delay=delay):
          with attempt:
              last_status = get_export_row(
                  source_table=source_table,
                  partition_id=partition_id,
                  destination=destination,
                  destination_table=destination_table,
                  columns="status",
                  node=node,
              )
              assert last_status == expected_status, error(

Description
  Expected status 'COMPLETED' for source_table='mt_912ddea3_a196_11f1_93b7_920009ae2786' partition_id='2020' but got 'FAILED'

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/export_status.py', line 218 in 'wait_for_export_status'

210\|              last_status = get_export_row(
211\|                  source_table=source_table,
212\|                  partition_id=partition_id,
213\|                  destination=destination,
214\|                  destination_table=destination_table,
215\|                  columns="status",
216\|                  node=node,
217\|              )
218\|>             assert last_status == expected_status, error(
219\|                  f"Expected status '{expected_status}' for "
220\|                  f"source_table='{source_table}' partition_id='{partition_id}'"
221\|                  f" but got '{last_status}'"