Merge branch 'apache:main' into hive_view

.baseline/checkstyle/checkstyle-suppressions.xml

@@ -2,6 +2,22 @@
 <!DOCTYPE suppressions PUBLIC
     "-//Checkstyle//DTD SuppressionFilter Configuration 1.2//EN"
     "https://checkstyle.org/dtds/suppressions_1_2.dtd">
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         https://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
 
 <!-- IMPORTANT ECLIPSE NOTE: If you change this file, you must restart Eclipse
  for your changes to take effect in its Checkstyle integration. -->

.baseline/checkstyle/checkstyle.xml

@@ -2,7 +2,22 @@
 <!DOCTYPE module PUBLIC
         "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
         "https://checkstyle.org/dtds/configuration_1_3.dtd">
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
 
+         https://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
 <module name="Checker">
     <property name="charset" value="UTF-8"/>
     <property name="severity" value="error"/>

.baseline/idea/intellij-java-palantir-style.xml

@@ -1,3 +1,19 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         https://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
 <project version="4">
   <component name="ProjectCodeStyleSettingsManager">
     <option name="PER_PROJECT_SETTINGS">

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -9,7 +9,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "1.4.0 (latest release)"
+        - "1.4.1 (latest release)"
+        - "1.4.0"
         - "1.3.1"
         - "1.3.0"
         - "1.2.1"

.github/ISSUE_TEMPLATE/iceberg_question.yml

@@ -5,7 +5,7 @@ labels: ["kind:question"]
 body:
   - type: markdown
     attributes:
-      value: "Feel free to ask your question on [Slack](https://join.slack.com/t/apache-iceberg/shared_invite/zt-1znkcg5zm-7_FE~pcox347XwZE3GNfPg) as well."
+      value: "Feel free to ask your question on [Slack](https://join.slack.com/t/apache-iceberg/shared_invite/zt-2561tq9qr-UtISlHgsdY3Virs3Z2_btQ) as well."
   - type: textarea
     attributes:
       label: Query engine

.github/workflows/delta-conversion-ci.yml

@@ -27,7 +27,7 @@ on:
       - 'apache-iceberg-**'
   pull_request:
     paths-ignore:
-      - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
+      - '.github/ISSUE_TEMPLATE/**'
       - '.github/workflows/flink-ci.yml'
       - '.github/workflows/hive-ci.yml'
       - '.gitignore'

.github/workflows/flink-ci.yml

@@ -27,7 +27,7 @@ on:
     - 'apache-iceberg-**'
   pull_request:
     paths-ignore:
-    - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
+    - '.github/ISSUE_TEMPLATE/**'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - '.gitignore'

.github/workflows/hive-ci.yml

@@ -27,7 +27,7 @@ on:
     - 'apache-iceberg-**'
   pull_request:
     paths-ignore:
-    - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
+    - '.github/ISSUE_TEMPLATE/**'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/flink-ci.yml'
     - '.gitignore'

.github/workflows/java-ci.yml

@@ -27,7 +27,7 @@ on:
     - 'apache-iceberg-**'
   pull_request:
     paths-ignore:
-    - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
+    - '.github/ISSUE_TEMPLATE/**'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'

.github/workflows/jmh-benchmarks.yml

@@ -50,8 +50,8 @@ jobs:
         run: |
           matrix=$(echo '[${{ github.event.inputs.benchmarks }}]' | jq '.[] | select(endswith("Benchmark")) | .')
           matrix=$(echo $matrix | sed 's/ /,/g' | sed 's/"/\"/g')
-          echo "::set-output name=matrix::[$matrix]"
-          echo "::set-output name=foundlabel::$(echo "[$matrix]" | jq 'if . | length > 0 then true else false end')"
+          echo "matrix=[$matrix]" >> $GITHUB_OUTPUT
+          echo "foundlabel=$(echo "[$matrix]" | jq 'if . | length > 0 then true else false end')" >> $GITHUB_OUTPUT
 
   show-matrix:
     needs: matrix

.github/workflows/spark-ci.yml

@@ -27,7 +27,7 @@ on:
     - 'apache-iceberg-**'
   pull_request:
     paths-ignore:
-    - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
+    - '.github/ISSUE_TEMPLATE/**'
     - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - '.gitignore'

.gitignore

@@ -2,7 +2,6 @@
 .DS_Store
 .cache
 tmp/
-site
 
 # intellij files
 .idea

README.md

@@ -37,7 +37,7 @@ The [Iceberg format specification][iceberg-spec] is being actively updated and i
 
 [Java API javadocs][iceberg-javadocs] are available for the main.
 
-[iceberg-javadocs]: https://iceberg.apache.org/javadoc/main
+[iceberg-javadocs]: https://iceberg.apache.org/javadoc/latest
 [iceberg-spec]: https://iceberg.apache.org/spec
 
 
@@ -78,7 +78,7 @@ Iceberg also has modules for adding Iceberg support to processing engines:
 * `iceberg-pig` is an implementation of Pig's LoadFunc API for Iceberg
 
 ---
-**NOTE** 
+**NOTE**
 
 The tests require Docker to execute. On MacOS (with Docker Desktop), you might need to create a symbolic name to the docker socket in order to be detected by the tests:
 

api/src/main/java/org/apache/iceberg/view/ViewVersion.java

@@ -65,7 +65,7 @@ public interface ViewVersion {
    * @return the string operation which produced the view version
    */
   default String operation() {
-    return summary().get("operation");
+    return versionId() == 1 ? "create" : "replace";
   }
 
   /** The query output schema at version create time, without aliases */

aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java

@@ -281,6 +281,7 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
                 .build());
     String dbLocationUri = response.database().locationUri();
     if (dbLocationUri != null) {
+      dbLocationUri = LocationUtil.stripTrailingSlash(dbLocationUri);
       return String.format("%s/%s", dbLocationUri, tableIdentifier.name());
     }
 
@@ -514,7 +515,9 @@ public Map<String, String> loadNamespaceMetadata(Namespace namespace)
       Map<String, String> result = Maps.newHashMap(database.parameters());
 
       if (database.locationUri() != null) {
-        result.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, database.locationUri());
+        result.put(
+            IcebergToGlueConverter.GLUE_DB_LOCATION_KEY,
+            LocationUtil.stripTrailingSlash(database.locationUri()));
       }
 
       if (database.description() != null) {

aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java

@@ -152,6 +152,19 @@ public void testDefaultWarehouseLocationDbUri() {
     Assertions.assertThat(location).isEqualTo("s3://bucket2/db/table");
   }
 
+  @Test
+  public void testDefaultWarehouseLocationDbUriTrailingSlash() {
+    Mockito.doReturn(
+            GetDatabaseResponse.builder()
+                .database(Database.builder().name("db").locationUri("s3://bucket2/db/").build())
+                .build())
+        .when(glue)
+        .getDatabase(Mockito.any(GetDatabaseRequest.class));
+    String location = glueCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"));
+
+    Assertions.assertThat(location).isEqualTo("s3://bucket2/db/table");
+  }
+
   @Test
   public void testDefaultWarehouseLocationCustomCatalogId() {
     GlueCatalog catalogWithCustomCatalogId = new GlueCatalog();
@@ -478,9 +491,15 @@ public void testListNamespacesBadName() {
   public void testLoadNamespaceMetadata() {
     Map<String, String> parameters = Maps.newHashMap();
     parameters.put("key", "val");
+    parameters.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://bucket2/db");
     Mockito.doReturn(
             GetDatabaseResponse.builder()
-                .database(Database.builder().name("db1").parameters(parameters).build())
+                .database(
+                    Database.builder()
+                        .name("db1")
+                        .parameters(parameters)
+                        .locationUri("s3://bucket2/db/")
+                        .build())
                 .build())
         .when(glue)
         .getDatabase(Mockito.any(GetDatabaseRequest.class));

core/src/main/java/org/apache/iceberg/BaseScan.java

@@ -48,7 +48,8 @@ abstract class BaseScan<ThisT, T extends ScanTask, G extends ScanTaskGroup<T>>
           "record_count",
           "partition",
           "key_metadata",
-          "split_offsets");
+          "split_offsets",
+          "sort_order_id");
 
   private static final List<String> STATS_COLUMNS =
       ImmutableList.of(

core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java

@@ -165,6 +165,10 @@ public void renameTable(TableIdentifier from, TableIdentifier to) {
         throw new AlreadyExistsException("Cannot rename %s to %s. Table already exists", from, to);
       }
 
+      if (views.containsKey(to)) {
+        throw new AlreadyExistsException("Cannot rename %s to %s. View already exists", from, to);
+      }
+
       tables.put(to, fromLocation);
       tables.remove(from);
     }
@@ -404,6 +408,10 @@ public void doCommit(TableMetadata base, TableMetadata metadata) {
                   throw new AlreadyExistsException("Table already exists: %s", tableName());
                 }
 
+                if (null == existingLocation) {
+                  throw new NoSuchTableException("Table does not exist: %s", tableName());
+                }
+
                 throw new CommitFailedException(
                     "Cannot commit to table %s metadata location from %s to %s "
                         + "because it has been concurrently modified to %s",
@@ -470,6 +478,10 @@ public void doCommit(ViewMetadata base, ViewMetadata metadata) {
                   throw new AlreadyExistsException("View already exists: %s", identifier);
                 }
 
+                if (null == existingLocation) {
+                  throw new NoSuchViewException("View does not exist: %s", identifier);
+                }
+
                 throw new CommitFailedException(
                     "Cannot commit to view %s metadata location from %s to %s "
                         + "because it has been concurrently modified to %s",

core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java

@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.EnvironmentContext;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.catalog.TableIdentifier;
@@ -161,7 +162,7 @@ private View create(ViewOperations ops) {
               .defaultNamespace(defaultNamespace)
               .defaultCatalog(defaultCatalog)
               .timestampMillis(System.currentTimeMillis())
-              .putSummary("operation", "create")
+              .putAllSummary(EnvironmentContext.get())
               .build();
 
       ViewMetadata viewMetadata =
@@ -206,7 +207,7 @@ private View replace(ViewOperations ops) {
               .defaultNamespace(defaultNamespace)
               .defaultCatalog(defaultCatalog)
               .timestampMillis(System.currentTimeMillis())
-              .putSummary("operation", "replace")
+              .putAllSummary(EnvironmentContext.get())
               .build();
 
       ViewMetadata.Builder builder =

core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java

@@ -19,7 +19,6 @@
 package org.apache.iceberg.view;
 
 import javax.annotation.Nullable;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.immutables.value.Value;
 
 /**
@@ -40,9 +39,7 @@ interface BaseViewVersion extends ViewVersion {
   @Override
   @Value.Lazy
   default String operation() {
-    Preconditions.checkArgument(
-        summary().containsKey("operation"), "Invalid view version summary, missing operation");
-    return summary().get("operation");
+    return ViewVersion.super.operation();
   }
 
   @Override

core/src/main/java/org/apache/iceberg/view/ViewMetadata.java

@@ -320,14 +320,15 @@ private int reuseOrCreateNewViewVersionId(ViewVersion viewVersion) {
 
     /**
      * Checks whether the given view versions would behave the same while ignoring the view version
-     * id, the creation timestamp, and the summary.
+     * id, the creation timestamp, and the operation.
      *
      * @param one the view version to compare
      * @param two the view version to compare
      * @return true if the given view versions would behave the same
      */
     private boolean sameViewVersion(ViewVersion one, ViewVersion two) {
-      return Objects.equals(one.representations(), two.representations())
+      return Objects.equals(one.summary(), two.summary())
+          && Objects.equals(one.representations(), two.representations())
           && Objects.equals(one.defaultCatalog(), two.defaultCatalog())
           && Objects.equals(one.defaultNamespace(), two.defaultNamespace())
           && one.schemaId() == two.schemaId();

core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java

@@ -37,6 +37,7 @@
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.iceberg.util.JsonUtil;
 
@@ -69,7 +70,9 @@ static void toJson(ViewMetadata metadata, JsonGenerator gen) throws IOException
     gen.writeStringField(VIEW_UUID, metadata.uuid());
     gen.writeNumberField(FORMAT_VERSION, metadata.formatVersion());
     gen.writeStringField(LOCATION, metadata.location());
-    JsonUtil.writeStringMap(PROPERTIES, metadata.properties(), gen);
+    if (!metadata.properties().isEmpty()) {
+      JsonUtil.writeStringMap(PROPERTIES, metadata.properties(), gen);
+    }
 
     gen.writeArrayFieldStart(SCHEMAS);
     for (Schema schema : metadata.schemas()) {
@@ -114,7 +117,8 @@ public static ViewMetadata fromJson(String metadataLocation, JsonNode json) {
     String uuid = JsonUtil.getString(VIEW_UUID, json);
     int formatVersion = JsonUtil.getInt(FORMAT_VERSION, json);
     String location = JsonUtil.getString(LOCATION, json);
-    Map<String, String> properties = JsonUtil.getStringMap(PROPERTIES, json);
+    Map<String, String> properties =
+        json.has(PROPERTIES) ? JsonUtil.getStringMap(PROPERTIES, json) : ImmutableMap.of();
 
     JsonNode schemasNode = JsonUtil.get(SCHEMAS, json);
 

core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java

@@ -28,6 +28,7 @@
 import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT;
 
 import java.util.List;
+import org.apache.iceberg.EnvironmentContext;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.exceptions.CommitFailedException;
@@ -77,7 +78,7 @@ private ViewMetadata internalApply() {
             .schemaId(schema.schemaId())
             .defaultNamespace(defaultNamespace)
             .defaultCatalog(defaultCatalog)
-            .putSummary("operation", "replace")
+            .putAllSummary(EnvironmentContext.get())
             .addAllRepresentations(representations)
             .build();