Iceberg Test Run Report

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 467, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 92, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 553, in feature
    Scenario(test=issue_with_decimal_column)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 459, in issue_with_decimal_column
    check_read_with_predicate_push_down(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 69, in check_read_with_predicate_push_down
    assert int(read_rows.output.strip()) == 0, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def check_read_with_predicate_push_down(
      self, where_clause, database_name, namespace, table_name
  ):
      """Check that ClickHouse does not read any rows when predicate pushdown is enabled
      and the condition does not match any data.
      """
      with By("drop all caches"):
          common.drop_all_caches(node=self.context.node)

      with And("read with input_format_parquet_filter_push_down"):
          log_comment_with_pruning = f"with_pruning_{getuid()}"
          result = iceberg_engine.read_data_from_clickhouse_iceberg_table(
              database_name=database_name,
              namespace=namespace,
              table_name=table_name,
              where_clause=where_clause,
              input_format_parquet_filter_push_down="1",
              log_comment=log_comment_with_pruning,
              use_iceberg_metadata_files_cache="0",
              use_iceberg_partition_pruning="0",
              input_format_parquet_bloom_filter_push_down="0"
          )
          assert result.output.strip() == "", error()

      with And(f"check that 0 rows were read"):
          read_rows = metrics.get_read_rows(
              log_comment=log_comment_with_pruning,
          )
          assert int(read_rows.output.strip()) == 0, error(

Description
  Expected 0 rows to be read, with where clause: decimal < 0.5, but got 10

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py', line 69 in 'check_read_with_predicate_push_down'

61\|              input_format_parquet_bloom_filter_push_down="0"
62\|          )
63\|          assert result.output.strip() == "", error()
64\|  
65\|      with And(f"check that 0 rows were read"):
66\|          read_rows = metrics.get_read_rows(
67\|              log_comment=log_comment_with_pruning,
68\|          )
69\|>         assert int(read_rows.output.strip()) == 0, error(
70\|              f"Expected 0 rows to be read, with where clause: {where_clause}, but got {read_rows.output.strip()}"
71\|          )
72\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 467, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 92, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 556, in feature
    Scenario(test=issue_with_float_column)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 529, in issue_with_float_column
    check_read_with_predicate_push_down(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py", line 69, in check_read_with_predicate_push_down
    assert int(read_rows.output.strip()) == 0, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def check_read_with_predicate_push_down(
      self, where_clause, database_name, namespace, table_name
  ):
      """Check that ClickHouse does not read any rows when predicate pushdown is enabled
      and the condition does not match any data.
      """
      with By("drop all caches"):
          common.drop_all_caches(node=self.context.node)

      with And("read with input_format_parquet_filter_push_down"):
          log_comment_with_pruning = f"with_pruning_{getuid()}"
          result = iceberg_engine.read_data_from_clickhouse_iceberg_table(
              database_name=database_name,
              namespace=namespace,
              table_name=table_name,
              where_clause=where_clause,
              input_format_parquet_filter_push_down="1",
              log_comment=log_comment_with_pruning,
              use_iceberg_metadata_files_cache="0",
              use_iceberg_partition_pruning="0",
              input_format_parquet_bloom_filter_push_down="0"
          )
          assert result.output.strip() == "", error()

      with And(f"check that 0 rows were read"):
          read_rows = metrics.get_read_rows(
              log_comment=log_comment_with_pruning,
          )
          assert int(read_rows.output.strip()) == 0, error(

Description
  Expected 0 rows to be read, with where clause: float < 0.5, but got 10

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/predicate_push_down.py', line 69 in 'check_read_with_predicate_push_down'

61\|              input_format_parquet_bloom_filter_push_down="0"
62\|          )
63\|          assert result.output.strip() == "", error()
64\|  
65\|      with And(f"check that 0 rows were read"):
66\|          read_rows = metrics.get_read_rows(
67\|              log_comment=log_comment_with_pruning,
68\|          )
69\|>         assert int(read_rows.output.strip()) == 0, error(
70\|              f"Expected 0 rows to be read, with where clause: {where_clause}, but got {read_rows.output.strip()}"
71\|          )
72\|

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 467, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 125, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 717, in feature
    Scenario(test=check_iceberg_partition_pruning_with_integer_type)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 161, in check_iceberg_partition_pruning_with_integer_type
    assert int(s3_read_requests_count) <= max_count, error()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert int(s3_read_requests_count) <= max_count, error()

Assertion values
  assert int(s3_read_requests_count) <= max_count, error()
             ^ is '74'
  assert int(s3_read_requests_count) <= max_count, error()
         ^ is = 74
  assert int(s3_read_requests_count) <= max_count, error()
                                        ^ is 53
  assert int(s3_read_requests_count) <= max_count, error()
                                     ^ is = False
  assert int(s3_read_requests_count) <= max_count, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py', line 161 in 'check_iceberg_partition_pruning_with_integer_type'

153\|                  # 2 requests for each row: one for metadata and one for data
154\|                  # 3-5 requests for metadata files
155\|                  s3_read_requests_count = metrics.get_S3ReadRequestsCount(
156\|                      log_comment=log_comment_with_partition_pruning,
157\|                  ).output.strip()
158\|                  min_count = 2 * (length - i) + 3
159\|                  max_count = 2 * (length - i) + 13
160\|                  assert min_count <= int(s3_read_requests_count), error()
161\|>                 assert int(s3_read_requests_count) <= max_count, error()
162\|  
163\|      with And("read data from ClickHouse with partition pruning disabled"):
164\|          for i in range(length):

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 467, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 125, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 720, in feature
    Scenario(test=check_partition_pruning_with_complex_where_clause)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 367, in check_partition_pruning_with_complex_where_clause
    assert int(s3_read_requests_count) <= 22, error()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert int(s3_read_requests_count) <= 22, error()

Assertion values
  assert int(s3_read_requests_count) <= 22, error()
             ^ is '27'
  assert int(s3_read_requests_count) <= 22, error()
         ^ is = 27
  assert int(s3_read_requests_count) <= 22, error()
                                     ^ is = False
  assert int(s3_read_requests_count) <= 22, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py', line 367 in 'check_partition_pruning_with_complex_where_clause'

359\|              assert int(min_max_pruned_files) == 43, error()
360\|  
361\|      with And("check that S3ReadRequestsCount is correct"):
362\|          s3_read_requests_count = metrics.get_S3ReadRequestsCount(
363\|              log_comment=log_comment_with_partition_pruning,
364\|          ).output.strip()
365\|          if check_clickhouse_version(">=25.4")(self) or check_if_antalya_build(self):
366\|              assert 10 <= int(s3_read_requests_count), error()
367\|>             assert int(s3_read_requests_count) <= 22, error()
368\|          else:
369\|              assert 95 <= int(s3_read_requests_count), error()
370\|              assert int(s3_read_requests_count) <= 100, error()

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 467, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/feature.py", line 125, in feature
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 723, in feature
    Scenario(test=partition_pruning_with_date_type)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py", line 512, in partition_pruning_with_date_type
    assert int(s3_read_requests_count) <= 150, error()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  assert int(s3_read_requests_count) <= 150, error()

Assertion values
  assert int(s3_read_requests_count) <= 150, error()
             ^ is '215'
  assert int(s3_read_requests_count) <= 150, error()
         ^ is = 215
  assert int(s3_read_requests_count) <= 150, error()
                                     ^ is = False
  assert int(s3_read_requests_count) <= 150, error()
  ^ is False

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/iceberg_engine/use_iceberg_partition_pruning.py', line 512 in 'partition_pruning_with_date_type'

504\|              assert int(min_max_pruned_files) == 0, error()
505\|  
506\|      with And("check that S3ReadRequestsCount is correct"):
507\|          s3_read_requests_count = metrics.get_S3ReadRequestsCount(
508\|              log_comment=log_comment_with_partition_pruning,
509\|          ).output.strip()
510\|          if check_clickhouse_version(">=25.3")(self) or check_if_antalya_build(self):
511\|              assert 137 <= int(s3_read_requests_count), error()
512\|>             assert int(s3_read_requests_count) <= 150, error()
513\|          else:
514\|              assert 200 <= int(s3_read_requests_count), error()
515\|              assert int(s3_read_requests_count) <= 210, error()

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 482, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 304, in value_counts_sum_to_row_count
    assert_value_counts_sum_to(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 561, in assert_value_counts_sum_to
    assert not bad, error(
           ^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_value_counts_sum_to(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_total,
  ):
      """For each column, assert ``sum(data_file.value_counts[field_id]) == expected_total``.

      This cross-checks the per-file statistics against the total row count
      without needing to decode ``lower_bounds`` / ``upper_bounds`` bytes.
      """
      table = load_pyiceberg_table(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      schema = table.schema()

      per_column_totals = {}
      for data_file in data_files:
          for field_id, count in (data_file.value_counts or {}).items():
              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count

      bad = []
      for field in schema.fields:
          total = per_column_totals.get(field.field_id, 0)
          if total != expected_total:
              bad.append((field.name, field.field_id, total))

      assert not bad, error(

Description
  Value_counts do not sum to 7 for iceberg_0664d17d_5e9b_11f1_af77_1232adabb401: [('id', 1, 0), ('year', 2, 0)]

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 561 in 'assert_value_counts_sum_to'

553\|              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count
554\|  
555\|      bad = []
556\|      for field in schema.fields:
557\|          total = per_column_totals.get(field.field_id, 0)
558\|          if total != expected_total:
559\|              bad.append((field.name, field.field_id, total))
560\|  
561\|>     assert not bad, error(
562\|          f"value_counts do not sum to {expected_total} for "
563\|          f"{as_destination_name(destination)}: {bad!r}"
564\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 428, in external_reader_round_trips_exported_data
    arrow_table = table.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1374, in _task_to_record_batches
    file_schema = pyarrow_to_schema(physical_schema, name_mapping, downcast_ns_timestamp_to_us=True)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 945, in pyarrow_to_schema
    raise ValueError(
ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 482, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 433, in external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_ManifestIntegrity_ExternalReader("1.0"))
  @Name("external iceberg reader round-trips exported data")
  def external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg (a non-ClickHouse reader) can scan the destination and
      materialise the exported rows.

      Currently XFail: EXPORT PARTITION writes Parquet without Iceberg
      field-ids and IcebergS3 tables do not set
      ``schema.name-mapping.default``, so strict readers (PyIceberg) cannot
      map physical columns to the table schema. On older builds that still
      write bucket-relative ``data_file.file_path`` URIs, the same scenario
      may instead fail at FileIO open with ``FileNotFoundError``.
      """
      source_table = f"mt_{getuid()}"
      expected_values = [(1, 2020), (2, 2020), (3, 2020)]
      values_sql = ", ".join(f"({id_}, {year})" for id_, year in expected_values)

      with Given("create source table and insert a known partition"):
          create_replicated_mergetree(
              table_name=source_table,
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
          )
          insert_data(table_name=source_table, values=values_sql)

      with And("create the Iceberg destination with full paths in metadata"):
          # write_full_path_in_iceberg_metadata = 1 is required infrastructure:
          # without it the manifest-list pointer in metadata.json is also
          # bucket-relative and PyIceberg fails before we ever reach a
          # data_file.file_path. We want the failure to be about the data
          # files, not the manifest list.
          destination = create_iceberg_destination(
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              query_settings=FULL_PATHS_SETTING,
          )

      with When("export the partition with full paths in metadata"):
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with And("load the destination via PyIceberg (external reader)"):
          table = load_pyiceberg_table(
              destination=destination,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with Then("PyIceberg can scan the table and materialise the rows"):
          try:
              arrow_table = table.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(

Description
  External reader (PyIceberg) could not map exported Parquet columns to the Iceberg table schema. EXPORT PARTITION writes Parquet without Iceberg field-ids, and IcebergS3 destinations do not set table property schema.name-mapping.default. Underlying error: ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py', line 433 in 'external_reader_round_trips_exported_data'

425\|  
426\|      with Then("PyIceberg can scan the table and materialise the rows"):
427\|          try:
428\|              arrow_table = table.scan().to_arrow()
429\|          except ValueError as exc:
430\|              msg = str(exc)
431\|              if "field-ids" not in msg and "name-mapping.default" not in msg:
432\|                  raise
433\|>             assert False, error(
434\|                  "External reader (PyIceberg) could not map exported Parquet "
435\|                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
436\|                  "Parquet without Iceberg field-ids, and IcebergS3 "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py", line 197, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py", line 183, in parquet_compression_method_flows_to_data_files
    assert normalised == {parquet_codec}, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_Settings_ParquetCompression("1.0"))
  @Name("output_format_parquet_compression_method flows to data files")
  def parquet_compression_method_flows_to_data_files(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """``output_format_parquet_compression_method`` set on ``ALTER ...
      EXPORT PARTITION`` reaches the Parquet writer (codec inspected via
      pyarrow). Currently XFail: the export-task settings allowlist drops
      format settings before ``getFormatSettings`` runs.
      """
      source_table = _seed_source()

      compressions = (("zstd", "ZSTD"), ("snappy", "SNAPPY"))

      for ch_codec, parquet_codec in compressions:
          with Given(f"create a dedicated Iceberg destination for {ch_codec}"):
              # write_full_path_in_iceberg_metadata = 1 so PyIceberg can follow
              # the manifest-list pointer in metadata.json via S3 (see the
              # FULL_PATHS_SETTING docstring). It does not influence the
              # compression codec under test.
              destination = create_iceberg_destination(
                  columns=SIMPLE_COLUMNS,
                  partition_by=SIMPLE_PARTITION_BY,
                  minio_root_user=minio_root_user,
                  minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  query_settings=FULL_PATHS_SETTING,
              )

          with When(f"export partition 2020 with {ch_codec} compression"):
              export_partition(
                  source_table=source_table,
                  destination=destination,
                  partition_id="2020",
                  extra_settings=FULL_PATHS_SETTING + [
                      ("output_format_parquet_compression_method", ch_codec),
                  ],
              )

          with Then(
              f"every column chunk of every {ch_codec} data file uses "
              f"{parquet_codec}"
          ):
              data_files = get_data_files(
                  destination=destination,
                  minio_root_user=minio_root_user,
                  minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              )
              assert data_files, error(
                  f"No data files found in destination for codec {ch_codec}"
              )

              observed = set()
              for data_file in data_files:
                  bucket, key = _parse_s3_file_path(
                      data_file.file_path, DEFAULT_S3_WAREHOUSE_BUCKET
                  )
                  observed \|= _read_parquet_compression_codecs(
                      bucket=bucket,
                      key=key,
                      minio_root_user=minio_root_user,
                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  )

              # pyarrow reports codec names in upper-case. We compare
              # case-insensitively to avoid coupling the test to that
              # convention.
              normalised = {c.upper() for c in observed}
              assert normalised == {parquet_codec}, error(

Description
  Expected every column to use 'SNAPPY', got ['ZSTD']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py', line 183 in 'parquet_compression_method_flows_to_data_files'

175\|                      minio_root_user=minio_root_user,
176\|                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
177\|                  )
178\|  
179\|              # pyarrow reports codec names in upper-case. We compare
180\|              # case-insensitively to avoid coupling the test to that
181\|              # convention.
182\|              normalised = {c.upper() for c in observed}
183\|>             assert normalised == {parquet_codec}, error(
184\|                  f"Expected every column to use {parquet_codec!r}, "
185\|                  f"got {sorted(observed)!r}"
186\|              )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 482, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 304, in value_counts_sum_to_row_count
    assert_value_counts_sum_to(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 561, in assert_value_counts_sum_to
    assert not bad, error(
           ^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_value_counts_sum_to(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_total,
  ):
      """For each column, assert ``sum(data_file.value_counts[field_id]) == expected_total``.

      This cross-checks the per-file statistics against the total row count
      without needing to decode ``lower_bounds`` / ``upper_bounds`` bytes.
      """
      table = load_pyiceberg_table(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      schema = table.schema()

      per_column_totals = {}
      for data_file in data_files:
          for field_id, count in (data_file.value_counts or {}).items():
              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count

      bad = []
      for field in schema.fields:
          total = per_column_totals.get(field.field_id, 0)
          if total != expected_total:
              bad.append((field.name, field.field_id, total))

      assert not bad, error(

Description
  Value_counts do not sum to 7 for datalake_7879cf5f_5e9b_11f1_a667_1232adabb401.\`export_ns_7879ceef_5e9b_11f1_b6ce_1232adabb401.iceberg_7879cf37_5e9b_11f1_8d39_1232adabb401\`: [('id', 1, 0), ('year', 2, 0)]

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 561 in 'assert_value_counts_sum_to'

553\|              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count
554\|  
555\|      bad = []
556\|      for field in schema.fields:
557\|          total = per_column_totals.get(field.field_id, 0)
558\|          if total != expected_total:
559\|              bad.append((field.name, field.field_id, total))
560\|  
561\|>     assert not bad, error(
562\|          f"value_counts do not sum to {expected_total} for "
563\|          f"{as_destination_name(destination)}: {bad!r}"
564\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 428, in external_reader_round_trips_exported_data
    arrow_table = table.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1374, in _task_to_record_batches
    file_schema = pyarrow_to_schema(physical_schema, name_mapping, downcast_ns_timestamp_to_us=True)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 945, in pyarrow_to_schema
    raise ValueError(
ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 482, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 433, in external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_ManifestIntegrity_ExternalReader("1.0"))
  @Name("external iceberg reader round-trips exported data")
  def external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg (a non-ClickHouse reader) can scan the destination and
      materialise the exported rows.

      Currently XFail: EXPORT PARTITION writes Parquet without Iceberg
      field-ids and IcebergS3 tables do not set
      ``schema.name-mapping.default``, so strict readers (PyIceberg) cannot
      map physical columns to the table schema. On older builds that still
      write bucket-relative ``data_file.file_path`` URIs, the same scenario
      may instead fail at FileIO open with ``FileNotFoundError``.
      """
      source_table = f"mt_{getuid()}"
      expected_values = [(1, 2020), (2, 2020), (3, 2020)]
      values_sql = ", ".join(f"({id_}, {year})" for id_, year in expected_values)

      with Given("create source table and insert a known partition"):
          create_replicated_mergetree(
              table_name=source_table,
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
          )
          insert_data(table_name=source_table, values=values_sql)

      with And("create the Iceberg destination with full paths in metadata"):
          # write_full_path_in_iceberg_metadata = 1 is required infrastructure:
          # without it the manifest-list pointer in metadata.json is also
          # bucket-relative and PyIceberg fails before we ever reach a
          # data_file.file_path. We want the failure to be about the data
          # files, not the manifest list.
          destination = create_iceberg_destination(
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              query_settings=FULL_PATHS_SETTING,
          )

      with When("export the partition with full paths in metadata"):
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with And("load the destination via PyIceberg (external reader)"):
          table = load_pyiceberg_table(
              destination=destination,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with Then("PyIceberg can scan the table and materialise the rows"):
          try:
              arrow_table = table.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(

Description
  External reader (PyIceberg) could not map exported Parquet columns to the Iceberg table schema. EXPORT PARTITION writes Parquet without Iceberg field-ids, and IcebergS3 destinations do not set table property schema.name-mapping.default. Underlying error: ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py', line 433 in 'external_reader_round_trips_exported_data'

425\|  
426\|      with Then("PyIceberg can scan the table and materialise the rows"):
427\|          try:
428\|              arrow_table = table.scan().to_arrow()
429\|          except ValueError as exc:
430\|              msg = str(exc)
431\|              if "field-ids" not in msg and "name-mapping.default" not in msg:
432\|                  raise
433\|>             assert False, error(
434\|                  "External reader (PyIceberg) could not map exported Parquet "
435\|                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
436\|                  "Parquet without Iceberg field-ids, and IcebergS3 "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 333, in catalog_external_reader_round_trips_exported_data
    arrow_table = external_view.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1374, in _task_to_record_batches
    file_schema = pyarrow_to_schema(physical_schema, name_mapping, downcast_ns_timestamp_to_us=True)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 945, in pyarrow_to_schema
    raise ValueError(
ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 392, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 338, in catalog_external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_CatalogIntegration_RestGlue("1.0"))
  @Name("catalog: external reader round-trips exported data")
  def catalog_external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg, following the catalog's metadata pointer, reads back
      the exported rows byte-exact. Catalog-mode analogue of the
      ``manifest_integrity`` external-reader scenario; skipped under
      ``no_catalog``.
      """
      source_table = _seed_source()

      with Given("materialise a catalog-backed Iceberg destination via PyIceberg"):
          destination = create_pyiceberg_catalog_destination(
              schema=CATALOG_SCHEMA,
              partition_spec=CATALOG_PARTITION_SPEC,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with When("export the single partition"):
          # ``destination=`` dispatches the catalog-aware split of
          # (destination_database, destination_table) when waiting for the
          # status to flip to ``COMPLETED``; see Phase 2 notes in
          # ``steps/export_status.py``.
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with Then("PyIceberg (external reader) can materialise the exported rows"):
          handle = as_pyiceberg_handle(destination)
          assert handle is not None, error(
              "Expected catalog-backed destination to expose a PyIceberg handle"
          )
          external_view = handle["pyiceberg_catalog"].load_table(
              f"{handle['namespace']}.{handle['table_name']}"
          )
          try:
              arrow_table = external_view.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(

Description
  External reader (PyIceberg) could not map exported Parquet columns to the Iceberg table schema. EXPORT PARTITION writes Parquet without Iceberg field-ids even when the catalog table was created with a proper schema. Underlying error: ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py', line 338 in 'catalog_external_reader_round_trips_exported_data'

330\|              f"{handle['namespace']}.{handle['table_name']}"
331\|          )
332\|          try:
333\|              arrow_table = external_view.scan().to_arrow()
334\|          except ValueError as exc:
335\|              msg = str(exc)
336\|              if "field-ids" not in msg and "name-mapping.default" not in msg:
337\|                  raise
338\|>             assert False, error(
339\|                  "External reader (PyIceberg) could not map exported Parquet "
340\|                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
341\|                  "Parquet without Iceberg field-ids even when the catalog "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py", line 197, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py", line 183, in parquet_compression_method_flows_to_data_files
    assert normalised == {parquet_codec}, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_Settings_ParquetCompression("1.0"))
  @Name("output_format_parquet_compression_method flows to data files")
  def parquet_compression_method_flows_to_data_files(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """``output_format_parquet_compression_method`` set on ``ALTER ...
      EXPORT PARTITION`` reaches the Parquet writer (codec inspected via
      pyarrow). Currently XFail: the export-task settings allowlist drops
      format settings before ``getFormatSettings`` runs.
      """
      source_table = _seed_source()

      compressions = (("zstd", "ZSTD"), ("snappy", "SNAPPY"))

      for ch_codec, parquet_codec in compressions:
          with Given(f"create a dedicated Iceberg destination for {ch_codec}"):
              # write_full_path_in_iceberg_metadata = 1 so PyIceberg can follow
              # the manifest-list pointer in metadata.json via S3 (see the
              # FULL_PATHS_SETTING docstring). It does not influence the
              # compression codec under test.
              destination = create_iceberg_destination(
                  columns=SIMPLE_COLUMNS,
                  partition_by=SIMPLE_PARTITION_BY,
                  minio_root_user=minio_root_user,
                  minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  query_settings=FULL_PATHS_SETTING,
              )

          with When(f"export partition 2020 with {ch_codec} compression"):
              export_partition(
                  source_table=source_table,
                  destination=destination,
                  partition_id="2020",
                  extra_settings=FULL_PATHS_SETTING + [
                      ("output_format_parquet_compression_method", ch_codec),
                  ],
              )

          with Then(
              f"every column chunk of every {ch_codec} data file uses "
              f"{parquet_codec}"
          ):
              data_files = get_data_files(
                  destination=destination,
                  minio_root_user=minio_root_user,
                  minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              )
              assert data_files, error(
                  f"No data files found in destination for codec {ch_codec}"
              )

              observed = set()
              for data_file in data_files:
                  bucket, key = _parse_s3_file_path(
                      data_file.file_path, DEFAULT_S3_WAREHOUSE_BUCKET
                  )
                  observed \|= _read_parquet_compression_codecs(
                      bucket=bucket,
                      key=key,
                      minio_root_user=minio_root_user,
                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  )

              # pyarrow reports codec names in upper-case. We compare
              # case-insensitively to avoid coupling the test to that
              # convention.
              normalised = {c.upper() for c in observed}
              assert normalised == {parquet_codec}, error(

Description
  Expected every column to use 'SNAPPY', got ['ZSTD']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py', line 183 in 'parquet_compression_method_flows_to_data_files'

175\|                      minio_root_user=minio_root_user,
176\|                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
177\|                  )
178\|  
179\|              # pyarrow reports codec names in upper-case. We compare
180\|              # case-insensitively to avoid coupling the test to that
181\|              # convention.
182\|              normalised = {c.upper() for c in observed}
183\|>             assert normalised == {parquet_codec}, error(
184\|                  f"Expected every column to use {parquet_codec!r}, "
185\|                  f"got {sorted(observed)!r}"
186\|              )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 482, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 304, in value_counts_sum_to_row_count
    assert_value_counts_sum_to(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py", line 561, in assert_value_counts_sum_to
    assert not bad, error(
           ^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestStep(Then)
  def assert_value_counts_sum_to(
      self,
      destination,
      minio_root_user,
      minio_root_[masked]:Secret(name='minio_root_password'),
      expected_total,
  ):
      """For each column, assert ``sum(data_file.value_counts[field_id]) == expected_total``.

      This cross-checks the per-file statistics against the total row count
      without needing to decode ``lower_bounds`` / ``upper_bounds`` bytes.
      """
      table = load_pyiceberg_table(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      data_files = get_data_files(
          destination=destination,
          minio_root_user=minio_root_user,
          minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
      )
      schema = table.schema()

      per_column_totals = {}
      for data_file in data_files:
          for field_id, count in (data_file.value_counts or {}).items():
              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count

      bad = []
      for field in schema.fields:
          total = per_column_totals.get(field.field_id, 0)
          if total != expected_total:
              bad.append((field.name, field.field_id, total))

      assert not bad, error(

Description
  Value_counts do not sum to 7 for datalake_a8ca5c63_5e9c_11f1_b016_1232adabb401.\`export_ns_a8ca5bfe_5e9c_11f1_b0f4_1232adabb401.iceberg_a8ca5c40_5e9c_11f1_817f_1232adabb401\`: [('id', 1, 0), ('year', 2, 0)]

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/steps/manifest_validation.py', line 561 in 'assert_value_counts_sum_to'

553\|              per_column_totals[field_id] = per_column_totals.get(field_id, 0) + count
554\|  
555\|      bad = []
556\|      for field in schema.fields:
557\|          total = per_column_totals.get(field.field_id, 0)
558\|          if total != expected_total:
559\|              bad.append((field.name, field.field_id, total))
560\|  
561\|>     assert not bad, error(
562\|          f"value_counts do not sum to {expected_total} for "
563\|          f"{as_destination_name(destination)}: {bad!r}"
564\|      )

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 428, in external_reader_round_trips_exported_data
    arrow_table = table.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1374, in _task_to_record_batches
    file_schema = pyarrow_to_schema(physical_schema, name_mapping, downcast_ns_timestamp_to_us=True)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 945, in pyarrow_to_schema
    raise ValueError(
ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 482, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py", line 433, in external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_ManifestIntegrity_ExternalReader("1.0"))
  @Name("external iceberg reader round-trips exported data")
  def external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg (a non-ClickHouse reader) can scan the destination and
      materialise the exported rows.

      Currently XFail: EXPORT PARTITION writes Parquet without Iceberg
      field-ids and IcebergS3 tables do not set
      ``schema.name-mapping.default``, so strict readers (PyIceberg) cannot
      map physical columns to the table schema. On older builds that still
      write bucket-relative ``data_file.file_path`` URIs, the same scenario
      may instead fail at FileIO open with ``FileNotFoundError``.
      """
      source_table = f"mt_{getuid()}"
      expected_values = [(1, 2020), (2, 2020), (3, 2020)]
      values_sql = ", ".join(f"({id_}, {year})" for id_, year in expected_values)

      with Given("create source table and insert a known partition"):
          create_replicated_mergetree(
              table_name=source_table,
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
          )
          insert_data(table_name=source_table, values=values_sql)

      with And("create the Iceberg destination with full paths in metadata"):
          # write_full_path_in_iceberg_metadata = 1 is required infrastructure:
          # without it the manifest-list pointer in metadata.json is also
          # bucket-relative and PyIceberg fails before we ever reach a
          # data_file.file_path. We want the failure to be about the data
          # files, not the manifest list.
          destination = create_iceberg_destination(
              columns=SIMPLE_COLUMNS,
              partition_by=SIMPLE_PARTITION_BY,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              query_settings=FULL_PATHS_SETTING,
          )

      with When("export the partition with full paths in metadata"):
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with And("load the destination via PyIceberg (external reader)"):
          table = load_pyiceberg_table(
              destination=destination,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with Then("PyIceberg can scan the table and materialise the rows"):
          try:
              arrow_table = table.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(

Description
  External reader (PyIceberg) could not map exported Parquet columns to the Iceberg table schema. EXPORT PARTITION writes Parquet without Iceberg field-ids, and IcebergS3 destinations do not set table property schema.name-mapping.default. Underlying error: ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/manifest_integrity.py', line 433 in 'external_reader_round_trips_exported_data'

425\|  
426\|      with Then("PyIceberg can scan the table and materialise the rows"):
427\|          try:
428\|              arrow_table = table.scan().to_arrow()
429\|          except ValueError as exc:
430\|              msg = str(exc)
431\|              if "field-ids" not in msg and "name-mapping.default" not in msg:
432\|                  raise
433\|>             assert False, error(
434\|                  "External reader (PyIceberg) could not map exported Parquet "
435\|                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
436\|                  "Parquet without Iceberg field-ids, and IcebergS3 "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 333, in catalog_external_reader_round_trips_exported_data
    arrow_table = external_view.scan().to_arrow()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/table/__init__.py", line 1763, in to_arrow
    ).to_table(self.plan_files())
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1575, in to_table
    if table_result := future.result():
                       ^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1556, in _table_from_scan_task
    batches = list(self._record_batches_from_scan_tasks_and_deletes([task], deletes_per_file))
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1637, in _record_batches_from_scan_tasks_and_deletes
    for batch in batches:
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 1374, in _task_to_record_batches
    file_schema = pyarrow_to_schema(physical_schema, name_mapping, downcast_ns_timestamp_to_us=True)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/venv/lib/python3.12/site-packages/pyiceberg/io/pyarrow.py", line 945, in pyarrow_to_schema
    raise ValueError(
ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 392, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py", line 338, in catalog_external_reader_round_trips_exported_data
    assert False, error(
           ^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_CatalogIntegration_RestGlue("1.0"))
  @Name("catalog: external reader round-trips exported data")
  def catalog_external_reader_round_trips_exported_data(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """PyIceberg, following the catalog's metadata pointer, reads back
      the exported rows byte-exact. Catalog-mode analogue of the
      ``manifest_integrity`` external-reader scenario; skipped under
      ``no_catalog``.
      """
      source_table = _seed_source()

      with Given("materialise a catalog-backed Iceberg destination via PyIceberg"):
          destination = create_pyiceberg_catalog_destination(
              schema=CATALOG_SCHEMA,
              partition_spec=CATALOG_PARTITION_SPEC,
              minio_root_user=minio_root_user,
              minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
          )

      with When("export the single partition"):
          # ``destination=`` dispatches the catalog-aware split of
          # (destination_database, destination_table) when waiting for the
          # status to flip to ``COMPLETED``; see Phase 2 notes in
          # ``steps/export_status.py``.
          export_partition(
              source_table=source_table,
              destination=destination,
              partition_id="2020",
              extra_settings=FULL_PATHS_SETTING,
          )

      with Then("PyIceberg (external reader) can materialise the exported rows"):
          handle = as_pyiceberg_handle(destination)
          assert handle is not None, error(
              "Expected catalog-backed destination to expose a PyIceberg handle"
          )
          external_view = handle["pyiceberg_catalog"].load_table(
              f"{handle['namespace']}.{handle['table_name']}"
          )
          try:
              arrow_table = external_view.scan().to_arrow()
          except ValueError as exc:
              msg = str(exc)
              if "field-ids" not in msg and "name-mapping.default" not in msg:
                  raise
              assert False, error(

Description
  External reader (PyIceberg) could not map exported Parquet columns to the Iceberg table schema. EXPORT PARTITION writes Parquet without Iceberg field-ids even when the catalog table was created with a proper schema. Underlying error: ValueError: Parquet file does not have field-ids and the Iceberg table does not have 'schema.name-mapping.default' defined

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/catalogs.py', line 338 in 'catalog_external_reader_round_trips_exported_data'

330\|              f"{handle['namespace']}.{handle['table_name']}"
331\|          )
332\|          try:
333\|              arrow_table = external_view.scan().to_arrow()
334\|          except ValueError as exc:
335\|              msg = str(exc)
336\|              if "field-ids" not in msg and "name-mapping.default" not in msg:
337\|                  raise
338\|>             assert False, error(
339\|                  "External reader (PyIceberg) could not map exported Parquet "
340\|                  "columns to the Iceberg table schema. EXPORT PARTITION writes "
341\|                  "Parquet without Iceberg field-ids even when the catalog "

AssertionError
Traceback (most recent call last):
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 497, in 
    regression()
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 179, in capture_cluster_args
    return func(self, cluster_args=cluster_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../helpers/argparser.py", line 450, in capture_minio_args
    return func(self, minio_args=minio_args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/regression.py", line 487, in regression
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 89, in feature
    _load_modules(self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password'))
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/feature.py", line 57, in _load_modules
    Feature(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py", line 197, in feature
    Scenario(test=scenario, flags=TE)(
  File "/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py", line 183, in parquet_compression_method_flows_to_data_files
    assert normalised == {parquet_codec}, error(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Oops! Assertion failed

The following assertion was not satisfied
  @TestScenario
  @Requirements(RQ_Iceberg_ExportPartition_Settings_ParquetCompression("1.0"))
  @Name("output_format_parquet_compression_method flows to data files")
  def parquet_compression_method_flows_to_data_files(
      self, minio_root_user, minio_root_[masked]:Secret(name='minio_root_password')
  ):
      """``output_format_parquet_compression_method`` set on ``ALTER ...
      EXPORT PARTITION`` reaches the Parquet writer (codec inspected via
      pyarrow). Currently XFail: the export-task settings allowlist drops
      format settings before ``getFormatSettings`` runs.
      """
      source_table = _seed_source()

      compressions = (("zstd", "ZSTD"), ("snappy", "SNAPPY"))

      for ch_codec, parquet_codec in compressions:
          with Given(f"create a dedicated Iceberg destination for {ch_codec}"):
              # write_full_path_in_iceberg_metadata = 1 so PyIceberg can follow
              # the manifest-list pointer in metadata.json via S3 (see the
              # FULL_PATHS_SETTING docstring). It does not influence the
              # compression codec under test.
              destination = create_iceberg_destination(
                  columns=SIMPLE_COLUMNS,
                  partition_by=SIMPLE_PARTITION_BY,
                  minio_root_user=minio_root_user,
                  minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  query_settings=FULL_PATHS_SETTING,
              )

          with When(f"export partition 2020 with {ch_codec} compression"):
              export_partition(
                  source_table=source_table,
                  destination=destination,
                  partition_id="2020",
                  extra_settings=FULL_PATHS_SETTING + [
                      ("output_format_parquet_compression_method", ch_codec),
                  ],
              )

          with Then(
              f"every column chunk of every {ch_codec} data file uses "
              f"{parquet_codec}"
          ):
              data_files = get_data_files(
                  destination=destination,
                  minio_root_user=minio_root_user,
                  minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
              )
              assert data_files, error(
                  f"No data files found in destination for codec {ch_codec}"
              )

              observed = set()
              for data_file in data_files:
                  bucket, key = _parse_s3_file_path(
                      data_file.file_path, DEFAULT_S3_WAREHOUSE_BUCKET
                  )
                  observed \|= _read_parquet_compression_codecs(
                      bucket=bucket,
                      key=key,
                      minio_root_user=minio_root_user,
                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
                  )

              # pyarrow reports codec names in upper-case. We compare
              # case-insensitively to avoid coupling the test to that
              # convention.
              normalised = {c.upper() for c in observed}
              assert normalised == {parquet_codec}, error(

Description
  Expected every column to use 'SNAPPY', got ['ZSTD']

Where
  File '/home/ubuntu/_work/ClickHouse/ClickHouse/iceberg/../iceberg/tests/export_partition/settings.py', line 183 in 'parquet_compression_method_flows_to_data_files'

175\|                      minio_root_user=minio_root_user,
176\|                      minio_root_[masked]:Secret(name='minio_root_password')=minio_root_[masked]:Secret(name='minio_root_password'),
177\|                  )
178\|  
179\|              # pyarrow reports codec names in upper-case. We compare
180\|              # case-insensitively to avoid coupling the test to that
181\|              # convention.
182\|              normalised = {c.upper() for c in observed}
183\|>             assert normalised == {parquet_codec}, error(
184\|                  f"Expected every column to use {parquet_codec!r}, "
185\|                  f"got {sorted(observed)!r}"
186\|              )