CREATE TABLE otto.test_table03 (
    `s1` STRUCT<`s1_f1`: STRING>
)
USING ICEBERG;

ALTER TABLE otto.test_table03 CHANGE COLUMN `s1` `s1` STRUCT<`s1_f1`: STRING, `s1_added_field0`: STRING>;

Error in query: Cannot update spark_catalog.otto.test_table03 field s1 type: update a struct by updating its fields; line 1 pos 0;
AlterTable org.apache.iceberg.spark.SparkSessionCatalog@1af78e37, otto.test_table03, RelationV2[s1#78] spark_catalog.otto.test_table03, [org.apache.spark.sql.connector.catalog.TableChange$UpdateColumnType@346f80ce]

spark-sql (default)> ALTER TABLE otto.eventerror ADD COLUMN event.added_field0 STRING;
Error in query: AlterTableAddColumnsCommand does not support nested column: event.added_field0

-- using 'hive'
CREATE TABLE otto.test_table04 (
  `s1` STRUCT<`s1_f1`: STRING COMMENT 's1_f1 is inside of struct s1'>
)
USING hive OPTIONS(fileFormat 'parquet') 
PARTITIONED BY (
  `f1` STRING
);
ALTER TABLE otto.test_table04 ADD COLUMN s1.added_field0 STRING;

-- using 'parquet'
CREATE TABLE otto.test_table05 (
  `s1` STRUCT<`s1_f1`: STRING COMMENT 's1_f1 is inside of struct s1'>
)
USING parquet
PARTITIONED BY (
  `f1` STRING
);
ALTER TABLE otto.test_table05 ADD COLUMN s1.added_field0 STRING;

CREATE TABLE otto.test_table06 (
  `s1` STRUCT<`s1_f1`: STRING COMMENT 's1_f1 is inside of struct s1'>
)
USING iceberg
PARTITIONED BY (
  `f1` STRING
);
ALTER TABLE otto.test_table06 ADD COLUMN s1.added_field0 STRING;

spark-sql (default)> describe otto.test_table06;
col_name	data_type	comment
s1	struct<s1_f1:string,added_field0:string>
f1	string

scala> spark.sql("ALTER TABLE otto.test_table04 ADD COLUMN s1.added_field0 STRING")

org.apache.spark.sql.AnalysisException: [UNSUPPORTED_FEATURE.TABLE_OPERATION] The feature is not supported: Table `spark_catalog`.`otto`.`test_table04` does not support ADD COLUMN with qualified column. Please check the current catalog and namespace to make sure the qualified table name is expected, and also check the catalog implementation which is configured by "spark.sql.catalog".

scala> spark.sql("ALTER TABLE otto.test_table05 ADD COLUMN s1.added_field0 STRING")

org.apache.spark.sql.AnalysisException: [UNSUPPORTED_FEATURE.TABLE_OPERATION] The feature is not supported: Table `spark_catalog`.`otto`.`test_table05` does not support ADD COLUMN with qualified column. Please check the current catalog and namespace to make sure the qualified table name is expected, and also check the catalog implementation which is configured by "spark.sql.catalog".

spark.sql("""
SELECT *
FROM xcollazo.mediawiki_content_history_v1
""").printSchema()
root
 |-- page_id: long (nullable = true)
...
 |-- revision_content_slots: map (nullable = true)
 |    |-- key: string
 |    |-- value: struct (valueContainsNull = true)
 |    |    |-- content_body: string (nullable = true)
 |    |    |-- content_format: string (nullable = true)
 |    |    |-- content_model: string (nullable = true)
 |    |    |-- content_sha1: string (nullable = true)
 |    |    |-- content_size: long (nullable = true)
 |-- revision_content_is_visible: boolean (nullable = true)
 |-- wiki_id: string (nullable = true)
...

spark.sql("""
-- add a field to the struct within an array. Using keyword 'value' to access the array's element column.
ALTER TABLE xcollazo.mediawiki_content_history_v1
ADD COLUMN revision_content_slots.value.origin_rev_id bigint;
""").show()

spark.sql("""
SELECT *
FROM xcollazo.mediawiki_content_history_v1
""").printSchema()
root
 |-- page_id: long (nullable = true)
...
 |-- revision_content_slots: map (nullable = true)
 |    |-- key: string
 |    |-- value: struct (valueContainsNull = true)
 |    |    |-- content_body: string (nullable = true)
 |    |    |-- content_format: string (nullable = true)
 |    |    |-- content_model: string (nullable = true)
 |    |    |-- content_sha1: string (nullable = true)
 |    |    |-- content_size: long (nullable = true)
 |    |    |-- origin_rev_id: long (nullable = true)
 |-- revision_content_is_visible: boolean (nullable = true)
 |-- wiki_id: string (nullable = true)
...


spark.sql("""
SELECT revision_content_slots['main'].content_size, revision_content_slots['main'].origin_rev_id
FROM xcollazo.mediawiki_content_history_v1
LIMIT 5
""").show()
[Stage 13:=====================================================>(190 + 1) / 191]
+-----------------------------------------+------------------------------------------+
|revision_content_slots[main].content_size|revision_content_slots[main].origin_rev_id|
+-----------------------------------------+------------------------------------------+
|                                     3867|                                      null|
|                                    18782|                                      null|
|                                    11710|                                      null|
|                                     2655|                                      null|
|                                     2825|                                      null|
+-----------------------------------------+------------------------------------------+

spark.sql("""
SELECT *
FROM xcollazo.mediawiki_content_history_v1
""").printSchema()

25/03/13 13:44:59 WARN TaskSetManager: Lost task 19.0 in stage 16.0 (TID 824) (an-worker1119.eqiad.wmnet executor 46): java.lang.IllegalArgumentException: [revision_content_slots, key_value, key] required binary key (STRING) = 22 is not in the store: [] 276000
	at org.apache.iceberg.shaded.org.apache.parquet.hadoop.ColumnChunkPageReadStore.getPageReader(ColumnChunkPageReadStore.java:288)
	at org.apache.iceberg.parquet.ParquetValueReaders$PrimitiveReader.setPageSource(ParquetValueReaders.java:224)
	at org.apache.iceberg.parquet.ParquetValueReaders$RepeatedKeyValueReader.setPageSource(ParquetValueReaders.java:572)
	at org.apache.iceberg.parquet.ParquetValueReaders$OptionReader.setPageSource(ParquetValueReaders.java:408)
	at org.apache.iceberg.parquet.ParquetValueReaders$StructReader.setPageSource(ParquetValueReaders.java:724)
	at org.apache.iceberg.parquet.ParquetReader$FileIterator.advance(ParquetReader.java:156)
	at org.apache.iceberg.parquet.ParquetReader$FileIterator.next(ParquetReader.java:126)
	at org.apache.iceberg.io.FilterIterator.advance(FilterIterator.java:65)
	at org.apache.iceberg.io.FilterIterator.hasNext(FilterIterator.java:49)
	at org.apache.iceberg.spark.source.BaseReader.next(BaseReader.java:135)
	at org.apache.spark.sql.execution.datasources.v2.PartitionIterator.hasNext(DataSourceRDD.scala:119)
	at org.apache.spark.sql.execution.datasources.v2.MetricsIterator.hasNext(DataSourceRDD.scala:156)
	at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1(DataSourceRDD.scala:63)
	at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.$anonfun$hasNext$1$adapted(DataSourceRDD.scala:63)
	at scala.Option.exists(Option.scala:376)
	at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:63)
	at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.advanceToNextIter(DataSourceRDD.scala:97)
	at org.apache.spark.sql.execution.datasources.v2.DataSourceRDD$$anon$1.hasNext(DataSourceRDD.scala:63)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:760)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:140)
	at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)
	at org.apache.spark.scheduler.Task.run(Task.scala:136)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:750)

ADD COLUMN revision_content_slots.value.origin_rev_id bigint;

spark.sql("""
CREATE TABLE xcollazo.test_add_column_to_array (
  a array<struct<one: string, two: bigint>>
)
USING ICEBERG
""").show()
++
||
++
++

spark.sql("""
INSERT INTO xcollazo.test_add_column_to_array
VALUES array(struct('1', 1)), array(struct('2', 2))
""").show()                                                                 
++
||
++
++

spark.sql("""
SELECT * FROM xcollazo.test_add_column_to_array
""").show()
+--------+
|       a|
+--------+
|[{1, 1}]|
|[{2, 2}]|
+--------+

                                                                                
spark.sql("""
ALTER TABLE xcollazo.test_add_column_to_array
ADD COLUMN a.element.three string
""").show()
++
||
++
++

spark.sql("""
SELECT * FROM xcollazo.test_add_column_to_array
""").show()
+--------------+
|             a|
+--------------+
|[{1, 1, null}]|
|[{2, 2, null}]|
+--------------+

                                                                                
spark.sql("""
SELECT a.two, a.three FROM xcollazo.test_add_column_to_array
""").show()
+---+------+
|two| three|
+---+------+
|[1]|[null]|
|[2]|[null]|
+---+------+

spark.sql("""
SELECT a.three, a.three IS NOT NULL FROM xcollazo.test_add_column_to_array
""").show()
+-----+---------------------+
|three|(a.three IS NOT NULL)|
+-----+---------------------+
|   []|                 true|                        <<<< BUG
|   []|                 true|
+-----+---------------------+

spark.sql("""
INSERT INTO xcollazo.test_add_column_to_array
VALUES array(struct('3', 3, '3'))
""").show()
++
||
++
++

spark.sql("""
SELECT * FROM xcollazo.test_add_column_to_array
""").show()
+--------------+
|             a|
+--------------+
|   [{3, 3, 3}]|
|[{1, 1, null}]|
|[{2, 2, null}]|
+--------------+

spark.sql("""
SELECT a.three FROM xcollazo.test_add_column_to_array
""").show()
+-----+
|three|
+-----+
|  [3]|
|   []|                        <<<< BUG
|   []|
+-----+