Flink: table supplier in writer operator for optional table refresh

core/src/main/java/org/apache/iceberg/io/OutputFileFactory.java

@@ -23,6 +23,7 @@
 
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 import org.apache.iceberg.FileFormat;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.StructLike;
@@ -35,7 +36,7 @@ public class OutputFileFactory {
   private final PartitionSpec defaultSpec;
   private final FileFormat format;
   private final LocationProvider locations;
-  private final FileIO io;
+  private final Supplier<FileIO> ioSupplier;
   private final EncryptionManager encryptionManager;
   private final int partitionId;
   private final long taskId;
@@ -56,7 +57,7 @@ public class OutputFileFactory {
    * @param spec Partition specification used by the location provider
    * @param format File format used for the extension
    * @param locations Location provider used for generating locations
-   * @param io FileIO to store the files
+   * @param ioSupplier Supplier of FileIO to store the files
    * @param encryptionManager Encryption manager used for encrypting the files
    * @param partitionId First part of the file name
    * @param taskId Second part of the file name
@@ -67,7 +68,7 @@ private OutputFileFactory(
       PartitionSpec spec,
       FileFormat format,
       LocationProvider locations,
-      FileIO io,
+      Supplier<FileIO> ioSupplier,
       EncryptionManager encryptionManager,
       int partitionId,
       long taskId,
@@ -76,7 +77,7 @@ private OutputFileFactory(
     this.defaultSpec = spec;
     this.format = format;
     this.locations = locations;
-    this.io = io;
+    this.ioSupplier = ioSupplier;
     this.encryptionManager = encryptionManager;
     this.partitionId = partitionId;
     this.taskId = taskId;
@@ -101,7 +102,7 @@ private String generateFilename() {
 
   /** Generates an {@link EncryptedOutputFile} for unpartitioned writes. */
   public EncryptedOutputFile newOutputFile() {
-    OutputFile file = io.newOutputFile(locations.newDataLocation(generateFilename()));
+    OutputFile file = ioSupplier.get().newOutputFile(locations.newDataLocation(generateFilename()));
     return encryptionManager.encrypt(file);
   }
 
@@ -113,7 +114,7 @@ public EncryptedOutputFile newOutputFile(StructLike partition) {
   /** Generates an {@link EncryptedOutputFile} for partitioned writes in a given spec. */
   public EncryptedOutputFile newOutputFile(PartitionSpec spec, StructLike partition) {
     String newDataLocation = locations.newDataLocation(spec, partition, generateFilename());
-    OutputFile rawOutputFile = io.newOutputFile(newDataLocation);
+    OutputFile rawOutputFile = ioSupplier.get().newOutputFile(newDataLocation);
     return encryptionManager.encrypt(rawOutputFile);
   }
 
@@ -125,6 +126,7 @@ public static class Builder {
     private String operationId;
     private FileFormat format;
     private String suffix;
+    private Supplier<FileIO> ioSupplier;
 
     private Builder(Table table, int partitionId, long taskId) {
       this.table = table;
@@ -136,6 +138,7 @@ private Builder(Table table, int partitionId, long taskId) {
       String formatAsString =
           table.properties().getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT);
       this.format = FileFormat.fromString(formatAsString);
+      this.ioSupplier = table::io;
     }
 
     public Builder defaultSpec(PartitionSpec newDefaultSpec) {
@@ -158,12 +161,31 @@ public Builder suffix(String newSuffix) {
       return this;
     }
 
+    /**
+     * Configures a {@link FileIO} supplier, which can potentially be used to dynamically refresh
+     * the file IO instance when a table is refreshed.
+     *
+     * @param newIoSupplier The file IO supplier
+     * @return this builder instance
+     */
+    public Builder ioSupplier(Supplier<FileIO> newIoSupplier) {
+      this.ioSupplier = newIoSupplier;
+      return this;
+    }
+
     public OutputFileFactory build() {
       LocationProvider locations = table.locationProvider();
-      FileIO io = table.io();
       EncryptionManager encryption = table.encryption();
       return new OutputFileFactory(
-          defaultSpec, format, locations, io, encryption, partitionId, taskId, operationId, suffix);
+          defaultSpec,
+          format,
+          locations,
+          ioSupplier,
+          encryption,
+          partitionId,
+          taskId,
+          operationId,
+          suffix);
     }
   }
 }

flink/v1.17/build.gradle

@@ -124,6 +124,11 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     }
 
     testImplementation libs.awaitility
+    testImplementation libs.assertj.core
+  }
+
+  test {
+    useJUnitPlatform()
   }
 }
 

flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/FlinkConfParser.java

@@ -18,11 +18,13 @@
  */
 package org.apache.iceberg.flink;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.util.TimeUtils;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
@@ -59,6 +61,10 @@ public StringConfParser stringConf() {
     return new StringConfParser();
   }
 
+  public DurationConfParser durationConf() {
+    return new DurationConfParser();
+  }
+
   class BooleanConfParser extends ConfParser<BooleanConfParser, Boolean> {
     private Boolean defaultValue;
 
@@ -180,6 +186,29 @@ public E parseOptional() {
     }
   }
 
+  class DurationConfParser extends ConfParser<DurationConfParser, Duration> {
+    private Duration defaultValue;
+
+    @Override
+    protected DurationConfParser self() {
+      return this;
+    }
+
+    public DurationConfParser defaultValue(Duration value) {
+      this.defaultValue = value;
+      return self();
+    }
+
+    public Duration parse() {
+      Preconditions.checkArgument(defaultValue != null, "Default value cannot be null");
+      return parse(TimeUtils::parseDuration, defaultValue);
+    }
+
+    public Duration parseOptional() {
+      return parse(TimeUtils::parseDuration, null);
+    }
+  }
+
   abstract class ConfParser<ThisT, T> {
     private final List<String> optionNames = Lists.newArrayList();
     private String tablePropertyName;

flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteConf.java

@@ -18,7 +18,9 @@
  */
 package org.apache.iceberg.flink;
 
+import java.time.Duration;
 import java.util.Map;
+import org.apache.calcite.linq4j.function.Experimental;
 import org.apache.flink.configuration.ReadableConfig;
 import org.apache.iceberg.DistributionMode;
 import org.apache.iceberg.FileFormat;
@@ -184,4 +186,20 @@ public String branch() {
   public Integer writeParallelism() {
     return confParser.intConf().option(FlinkWriteOptions.WRITE_PARALLELISM.key()).parseOptional();
   }
+
+  /**
+   * NOTE: This may be removed or changed in a future release. This value specifies the interval for
+   * refreshing the table instances in sink writer subtasks. If not specified then the default
+   * behavior is to not refresh the table.
+   *
+   * @return the interval for refreshing the table in sink writer subtasks
+   */
+  @Experimental
+  public Duration tableRefreshInterval() {
+    return confParser
+        .durationConf()
+        .option(FlinkWriteOptions.TABLE_REFRSH_INTERVAL.key())
+        .flinkConfig(FlinkWriteOptions.TABLE_REFRSH_INTERVAL)
+        .parseOptional();
+  }
 }

flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/FlinkWriteOptions.java

@@ -18,6 +18,8 @@
  */
 package org.apache.iceberg.flink;
 
+import java.time.Duration;
+import org.apache.calcite.linq4j.function.Experimental;
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ConfigOptions;
 import org.apache.iceberg.SnapshotRef;
@@ -64,4 +66,8 @@ private FlinkWriteOptions() {}
 
   public static final ConfigOption<Integer> WRITE_PARALLELISM =
       ConfigOptions.key("write-parallelism").intType().noDefaultValue();
+
+  @Experimental
+  public static final ConfigOption<Duration> TABLE_REFRSH_INTERVAL =
+      ConfigOptions.key("table-refresh-interval").durationType().noDefaultValue();
 }

flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/TableLoader.java

@@ -38,6 +38,8 @@ public interface TableLoader extends Closeable, Serializable, Cloneable {
 
   void open();
 
+  boolean isOpen();
+
   Table loadTable();
 
   /** Clone a TableLoader */
@@ -75,6 +77,11 @@ public void open() {
       tables = new HadoopTables(hadoopConf.get());
     }
 
+    @Override
+    public boolean isOpen() {
+      return tables != null;
+    }
+
     @Override
     public Table loadTable() {
       FlinkEnvironmentContext.init();
@@ -115,6 +122,11 @@ public void open() {
       catalog = catalogLoader.loadCatalog();
     }
 
+    @Override
+    public boolean isOpen() {
+      return catalog != null;
+    }
+
     @Override
     public Table loadTable() {
       FlinkEnvironmentContext.init();
@@ -126,6 +138,8 @@ public void close() throws IOException {
       if (catalog instanceof Closeable) {
         ((Closeable) catalog).close();
       }
+
+      catalog = null;
     }
 
     @Override

flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/sink/CachingTableSupplier.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.sink;
+
+import java.time.Duration;
+import org.apache.flink.util.Preconditions;
+import org.apache.iceberg.SerializableTable;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.flink.TableLoader;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.apache.iceberg.util.SerializableSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A table loader that will only reload a table after a certain interval has passed. WARNING: This
+ * table loader should be used carefully when used with writer tasks. It could result in heavy load
+ * on a catalog for jobs with many writers.
+ */
+class CachingTableSupplier implements SerializableSupplier<Table> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(CachingTableSupplier.class);
+
+  private final Table initialTable;
+  private final TableLoader tableLoader;
+  private final Duration tableRefreshInterval;
+  private long lastLoadTimeMillis;
+  private transient Table table;
+
+  CachingTableSupplier(
+      SerializableTable initialTable, TableLoader tableLoader, Duration tableRefreshInterval) {
+    Preconditions.checkArgument(initialTable != null, "initialTable cannot be null");
+    Preconditions.checkArgument(tableLoader != null, "tableLoader cannot be null");
+    Preconditions.checkArgument(
+        tableRefreshInterval != null, "tableRefreshInterval cannot be null");
+    this.initialTable = initialTable;
+    this.table = initialTable;
+    this.tableLoader = tableLoader;
+    this.tableRefreshInterval = tableRefreshInterval;
+    this.lastLoadTimeMillis = System.currentTimeMillis();
+  }
+
+  @Override
+  public Table get() {
+    if (table == null) {
+      this.table = initialTable;
+    }
+    return table;
+  }
+
+  Table initialTable() {
+    return initialTable;
+  }
+
+  void refreshTable() {
+    if (System.currentTimeMillis() > lastLoadTimeMillis + tableRefreshInterval.toMillis()) {
+      try {
+        if (!tableLoader.isOpen()) {
+          tableLoader.open();
+        }
+
+        this.table = tableLoader.loadTable();
+        this.lastLoadTimeMillis = System.currentTimeMillis();
+
+        LOG.info(
+            "Table {} reloaded, next min load time threshold is {}",
+            table.name(),
+            DateTimeUtil.formatTimestampMillis(
+                lastLoadTimeMillis + tableRefreshInterval.toMillis()));
+      } catch (Exception e) {
+        LOG.warn("An error occurred reloading table {}, table was not reloaded", table.name(), e);
+      }
+    }
+  }
+}

flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkManifestUtil.java

@@ -24,13 +24,11 @@
 import java.util.function.Supplier;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DeleteFile;
-import org.apache.iceberg.HasTableOperations;
 import org.apache.iceberg.ManifestFile;
 import org.apache.iceberg.ManifestFiles;
 import org.apache.iceberg.ManifestWriter;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Table;
-import org.apache.iceberg.TableOperations;
 import org.apache.iceberg.io.CloseableIterable;
 import org.apache.iceberg.io.FileIO;
 import org.apache.iceberg.io.OutputFile;
@@ -64,16 +62,14 @@ static List<DataFile> readDataFiles(
   }
 
   static ManifestOutputFileFactory createOutputFileFactory(
-      Table table, String flinkJobId, String operatorUniqueId, int subTaskId, long attemptNumber) {
-    TableOperations ops = ((HasTableOperations) table).operations();
+      Supplier<Table> tableSupplier,
+      Map<String, String> tableProps,
+      String flinkJobId,
+      String operatorUniqueId,
+      int subTaskId,
+      long attemptNumber) {
     return new ManifestOutputFileFactory(
-        ops,
-        table.io(),
-        table.properties(),
-        flinkJobId,
-        operatorUniqueId,
-        subTaskId,
-        attemptNumber);
+        tableSupplier, tableProps, flinkJobId, operatorUniqueId, subTaskId, attemptNumber);
   }
 
   /**