[AMORO-3256] Amoro Open Api with Swagger in Ams

amoro-ams/pom.xml

@@ -35,6 +35,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <skip-generate-sdk>true</skip-generate-sdk>
         <git-commit-id-plugin.fail-on-no-git-dir>false</git-commit-id-plugin.fail-on-no-git-dir>
     </properties>
 
@@ -279,7 +280,7 @@
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>url-connection-client</artifactId>
         </dependency>
-	
+
 	    <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>s3-transfer-manager</artifactId>
@@ -408,6 +409,18 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.webjars</groupId>
+            <artifactId>swagger-ui</artifactId>
+            <version>5.17.14</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.13</version>
+        </dependency>
+
         <!-- testcontainers dependencies -->
         <dependency>
             <groupId>org.testcontainers</groupId>
@@ -503,6 +516,69 @@
                     <skip>true</skip>
                 </configuration>
             </plugin>
+
+            <plugin>
+                <groupId>org.openapitools</groupId>
+                <artifactId>openapi-generator-maven-plugin</artifactId>
+                <version>6.0.0</version>
+                <executions>
+                    <execution>
+                        <id>generate-sdk</id>
+                        <goals>
+                            <goal>generate</goal>
+                        </goals>
+                        <phase>generate-sources</phase>
+                        <configuration>
+                            <inputSpec>${project.basedir}/src/main/resources/openapi/openapi.yaml</inputSpec>
+                            <generatorName>java</generatorName>
+                            <output>${project.build.directory}/generated-sources/openapi</output>
+                            <apiPackage>com.amoro.sdk.api</apiPackage>
+                            <modelPackage>com.amoro.sdk.model</modelPackage>
+                            <invokerPackage>com.amoro.sdk.invoker</invokerPackage>
+                            <configOptions>
+                                <library>apache-httpclient</library>
+                            </configOptions>
+                            <skip>${skip-generate-sdk}</skip>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>generate-doc</id>
+                        <goals>
+                            <goal>generate</goal>
+                        </goals>
+                        <phase>generate-sources</phase>
+                        <configuration>
+                            <inputSpec>${project.basedir}/src/main/resources/openapi/openapi.yaml</inputSpec>
+                            <generatorName>html2</generatorName>
+                            <output>${project.build.directory}/generated-docs/openapi</output>
+                            <skip>${skip-generate-sdk}</skip>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>package-sdk</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                        <phase>package</phase>
+                        <configuration>
+                            <classifier>sdk</classifier>
+                            <classesDirectory>${project.build.directory}/generated-sources/openapi/src/main/java</classesDirectory>
+                            <outputDirectory>${project.build.directory}</outputDirectory>
+                            <finalName>sdk</finalName>
+                        </configuration>
+                    </execution>
+                </executions>
+                <configuration>
+                    <skip>${skip-generate-sdk}</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
@@ -544,5 +620,11 @@
                 <git-commit-id-plugin.fail-on-no-git-dir>true</git-commit-id-plugin.fail-on-no-git-dir>
             </properties>
         </profile>
+        <profile>
+            <id>generate-sdk</id>
+            <properties>
+                <skip-generate-sdk>false</skip-generate-sdk>
+            </properties>
+        </profile>
     </profiles>
 </project>

amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java

@@ -20,6 +20,7 @@
 
 import io.javalin.Javalin;
 import io.javalin.http.HttpCode;
+import io.javalin.http.staticfiles.Location;
 import org.apache.amoro.Constants;
 import org.apache.amoro.OptimizerProperties;
 import org.apache.amoro.api.AmoroTableMetastore;
@@ -241,11 +242,14 @@ private void initHttpService() {
         Javalin.create(
             config -> {
               config.addStaticFiles(dashboardServer.configStaticFiles());
+              config.addStaticFiles("/META-INF/resources/webjars", Location.CLASSPATH);
               config.sessionHandler(SessionHandler::new);
               config.enableCorsForAllOrigins();
               config.jsonMapper(JavalinJsonMapper.createDefaultJsonMapper());
               config.showJavalinBanner = false;
+              config.enableWebjars();
             });
+
     httpServer.routes(
         () -> {
           dashboardServer.endpoints().addEndpoints();

amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java

@@ -76,6 +76,7 @@ public class DashboardServer {
   private static final String AUTH_TYPE_BASIC = "basic";
   private static final String X_REQUEST_SOURCE_HEADER = "X-Request-Source";
   private static final String X_REQUEST_SOURCE_WEB = "Web";
+  private static final String SWAGGER_PATH = "/swagger-ui";
 
   private final CatalogController catalogController;
   private final HealthCheckController healthCheckController;
@@ -118,7 +119,6 @@ public DashboardServer(
   }
 
   private String indexHtml = "";
-
   // read index.html content
   public String getIndexFileContent() {
     try {
@@ -170,6 +170,17 @@ public EndpointGroup endpoints() {
       path(
           "",
           () -> {
+            get(
+                "/swagger-docs",
+                ctx -> {
+                  InputStream openapiStream =
+                      getClass().getClassLoader().getResourceAsStream("openapi/openapi.yaml");
+                  if (openapiStream == null) {
+                    ctx.status(404).result("OpenAPI specification file not found");
+                  } else {
+                    ctx.result(openapiStream);
+                  }
+                });
             // static files
             get(
                 "/{page}",
@@ -358,6 +369,9 @@ private EndpointGroup apiGroup() {
   public void preHandleRequest(Context ctx) {
     String uriPath = ctx.path();
     String requestSource = ctx.header(X_REQUEST_SOURCE_HEADER);
+    if (uriPath.startsWith(SWAGGER_PATH)) {
+      return;
+    }
     if (needApiKeyCheck(uriPath) && !X_REQUEST_SOURCE_WEB.equalsIgnoreCase(requestSource)) {
       if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType)) {
         BasicAuthCredentials cred = ctx.basicAuthCredentials();
@@ -410,6 +424,9 @@ public void handleException(Exception e, Context ctx) {
     "/index.html",
     "/favicon.ico",
     "/assets/*",
+    "/swagger-ui",
+    "/swagger-ui/*",
+    "/swagger-docs",
     RestCatalogService.ICEBERG_REST_API_PREFIX + "/*"
   };
 

amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java

@@ -87,6 +87,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -341,7 +343,7 @@ private void collectSnapshots(
 
   @Override
   public List<PartitionFileBaseInfo> getSnapshotDetail(
-      AmoroTable<?> amoroTable, String snapshotId) {
+      AmoroTable<?> amoroTable, String snapshotId, @Nullable String ref) {
     MixedTable mixedTable = getTable(amoroTable);
     List<PartitionFileBaseInfo> result = new ArrayList<>();
     long commitId = Long.parseLong(snapshotId);

amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java

@@ -81,10 +81,10 @@ public List<AmoroSnapshotsOfTable> getSnapshots(
   }
 
   public List<PartitionFileBaseInfo> getSnapshotDetail(
-      TableIdentifier tableIdentifier, String snapshotId) {
+      TableIdentifier tableIdentifier, String snapshotId, String ref) {
     AmoroTable<?> amoroTable = loadTable(tableIdentifier);
     FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format());
-    return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId);
+    return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId, ref);
   }
 
   public List<DDLInfo> getTableOperations(TableIdentifier tableIdentifier) {

amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java

@@ -150,10 +150,12 @@ public void getTableDetail(Context ctx) {
                 TableIdentifier.of(catalog, database, tableName).buildTableIdentifier()));
     if (serverTableIdentifier.isPresent()) {
       TableRuntime tableRuntime = tableService.getRuntime(serverTableIdentifier.get().getId());
-      tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name());
-      OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary();
-      if (tableRuntimeSummary != null) {
-        tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore());
+      if (tableRuntime != null) {
+        tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name());
+        OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary();
+        if (tableRuntimeSummary != null) {
+          tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore());
+        }
       }
     } else {
       tableSummary.setOptimizingStatus(OptimizingStatus.IDLE.name());
@@ -416,10 +418,13 @@ public void getSnapshotDetail(Context ctx) {
     String snapshotId = ctx.pathParam("snapshotId");
     Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1);
     Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20);
+    String ref = ctx.queryParamAsClass("ref", String.class).getOrDefault(null);
 
     List<PartitionFileBaseInfo> result =
         tableDescriptor.getSnapshotDetail(
-            TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(), snapshotId);
+            TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(),
+            snapshotId,
+            ref);
     int offset = (page - 1) * pageSize;
     PageResult<PartitionFileBaseInfo> amsPageResult = PageResult.of(result, offset, pageSize);
     ctx.json(OkResponse.of(amsPageResult));

amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java

@@ -264,13 +264,13 @@ private TableOptimizingProcess planInternal(TableRuntime tableRuntime) {
     try {
       AmoroTable<?> table = tableManager.loadTable(tableRuntime.getTableIdentifier());
       OptimizingPlanner planner =
-          new OptimizingPlanner(
+          OptimizingPlanner.createOptimizingPlanner(
               tableRuntime.refresh(table),
               (MixedTable) table.originalTable(),
               getAvailableCore(),
               maxInputSizePerThread());
       if (planner.isNecessary()) {
-        return new TableOptimizingProcess(planner);
+        return new TableOptimizingProcess(planner, tableRuntime);
       } else {
         tableRuntime.completeEmptyProcess();
         return null;
@@ -371,9 +371,9 @@ public TaskRuntime poll() {
       }
     }
 
-    public TableOptimizingProcess(OptimizingPlanner planner) {
+    public TableOptimizingProcess(OptimizingPlanner planner, TableRuntime tableRuntime) {
       processId = planner.getProcessId();
-      tableRuntime = planner.getTableRuntime();
+      this.tableRuntime = tableRuntime;
       optimizingType = planner.getOptimizingType();
       planTime = planner.getPlanTime();
       targetSnapshotId = planner.getTargetSnapshotId();

amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/AbstractPartitionPlan.java

@@ -18,12 +18,12 @@
 
 package org.apache.amoro.server.optimizing.plan;
 
+import org.apache.amoro.ServerTableIdentifier;
 import org.apache.amoro.config.OptimizingConfig;
 import org.apache.amoro.optimizing.OptimizingInputProperties;
 import org.apache.amoro.optimizing.RewriteFilesInput;
 import org.apache.amoro.server.optimizing.OptimizingType;
 import org.apache.amoro.server.optimizing.RewriteStageTask;
-import org.apache.amoro.server.table.TableRuntime;
 import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
 import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
 import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
@@ -48,7 +48,9 @@ public abstract class AbstractPartitionPlan implements PartitionEvaluator {
 
   protected final Pair<Integer, StructLike> partition;
   protected final OptimizingConfig config;
-  protected final TableRuntime tableRuntime;
+  protected final ServerTableIdentifier identifier;
+  protected final long lastMinorOptimizingTime;
+  protected final long lastFullOptimizingTime;
   private CommonPartitionEvaluator evaluator;
   private TaskSplitter taskSplitter;
   protected MixedTable tableObject;
@@ -74,15 +76,20 @@ public abstract class AbstractPartitionPlan implements PartitionEvaluator {
   protected final Set<String> reservedDeleteFiles = Sets.newHashSet();
 
   public AbstractPartitionPlan(
-      TableRuntime tableRuntime,
+      ServerTableIdentifier identifier,
       MixedTable table,
+      OptimizingConfig config,
       Pair<Integer, StructLike> partition,
-      long planTime) {
+      long planTime,
+      long lastMinorOptimizingTime,
+      long lastFullOptimizingTime) {
+    this.identifier = identifier;
     this.partition = partition;
     this.tableObject = table;
-    this.config = tableRuntime.getOptimizingConfig();
-    this.tableRuntime = tableRuntime;
+    this.config = config;
     this.planTime = planTime;
+    this.lastMinorOptimizingTime = lastMinorOptimizingTime;
+    this.lastFullOptimizingTime = lastFullOptimizingTime;
   }
 
   @Override
@@ -98,7 +105,8 @@ protected CommonPartitionEvaluator evaluator() {
   }
 
   protected CommonPartitionEvaluator buildEvaluator() {
-    return new CommonPartitionEvaluator(tableRuntime, partition, planTime);
+    return new CommonPartitionEvaluator(
+        identifier, config, partition, planTime, lastMinorOptimizingTime, lastFullOptimizingTime);
   }
 
   @Override
@@ -317,10 +325,7 @@ public RewriteStageTask buildTask(OptimizingInputProperties properties) {
           MixedTableUtil.getMixedTablePartitionSpecById(tableObject, partition.first());
       String partitionPath = spec.partitionToPath(partition.second());
       return new RewriteStageTask(
-          tableRuntime.getTableIdentifier().getId(),
-          partitionPath,
-          input,
-          properties.getProperties());
+          identifier.getId(), partitionPath, input, properties.getProperties());
     }
   }