Merge branch 'master' into feature-swagger

apache · Nov 4, 2024 · 815c0b8 · 815c0b8
2 parents 926726b + fc744b1
commit 815c0b8
Show file tree

Hide file tree

Showing 13 changed files with 167 additions and 24 deletions.
diff --git a/...o-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java b/...o-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java
@@ -87,6 +87,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -341,7 +343,7 @@ private void collectSnapshots(
 
   @Override
   public List<PartitionFileBaseInfo> getSnapshotDetail(
-      AmoroTable<?> amoroTable, String snapshotId) {
+      AmoroTable<?> amoroTable, String snapshotId, @Nullable String ref) {
     MixedTable mixedTable = getTable(amoroTable);
     List<PartitionFileBaseInfo> result = new ArrayList<>();
     long commitId = Long.parseLong(snapshotId);

diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java
@@ -81,10 +81,10 @@ public List<AmoroSnapshotsOfTable> getSnapshots(
   }
 
   public List<PartitionFileBaseInfo> getSnapshotDetail(
-      TableIdentifier tableIdentifier, String snapshotId) {
+      TableIdentifier tableIdentifier, String snapshotId, String ref) {
     AmoroTable<?> amoroTable = loadTable(tableIdentifier);
     FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format());
-    return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId);
+    return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId, ref);
   }
 
   public List<DDLInfo> getTableOperations(TableIdentifier tableIdentifier) {

diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java
@@ -150,10 +150,12 @@ public void getTableDetail(Context ctx) {
                 TableIdentifier.of(catalog, database, tableName).buildTableIdentifier()));
     if (serverTableIdentifier.isPresent()) {
       TableRuntime tableRuntime = tableService.getRuntime(serverTableIdentifier.get().getId());
-      tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name());
-      OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary();
-      if (tableRuntimeSummary != null) {
-        tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore());
+      if (tableRuntime != null) {
+        tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name());
+        OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary();
+        if (tableRuntimeSummary != null) {
+          tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore());
+        }
       }
     } else {
       tableSummary.setOptimizingStatus(OptimizingStatus.IDLE.name());
@@ -416,10 +418,13 @@ public void getSnapshotDetail(Context ctx) {
     String snapshotId = ctx.pathParam("snapshotId");
     Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1);
     Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20);
+    String ref = ctx.queryParamAsClass("ref", String.class).getOrDefault(null);
 
     List<PartitionFileBaseInfo> result =
         tableDescriptor.getSnapshotDetail(
-            TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(), snapshotId);
+            TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(),
+            snapshotId,
+            ref);
     int offset = (page - 1) * pageSize;
     PageResult<PartitionFileBaseInfo> amsPageResult = PageResult.of(result, offset, pageSize);
     ctx.json(OkResponse.of(amsPageResult));

diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableRuntime.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableRuntime.java
@@ -51,21 +51,19 @@
 
 import java.math.BigDecimal;
 import java.math.RoundingMode;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 public class TableRuntime extends StatedPersistentBase {
 
   private static final Logger LOG = LoggerFactory.getLogger(TableRuntime.class);
 
   private final TableRuntimeHandler tableHandler;
   private final ServerTableIdentifier tableIdentifier;
-  private final List<TaskRuntime.TaskQuota> taskQuotas =
-      Collections.synchronizedList(new ArrayList<>());
+  private final List<TaskRuntime.TaskQuota> taskQuotas = new CopyOnWriteArrayList<>();
 
   // for unKeyedTable or base table
   @StateField private volatile long currentSnapshotId = AmoroServiceConstants.INVALID_SNAPSHOT_ID;

diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java
@@ -48,7 +48,8 @@ List<AmoroSnapshotsOfTable> getSnapshots(
       AmoroTable<?> amoroTable, String ref, OperationType operationType);
 
   /** Get the snapshot detail information of the {@link AmoroTable}. */
-  List<PartitionFileBaseInfo> getSnapshotDetail(AmoroTable<?> amoroTable, String snapshotId);
+  List<PartitionFileBaseInfo> getSnapshotDetail(
+      AmoroTable<?> amoroTable, String snapshotId, String ref);
 
   /** Get the DDL information of the {@link AmoroTable}. */
   List<DDLInfo> getTableOperations(AmoroTable<?> amoroTable);

diff --git a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java
@@ -74,6 +74,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.text.ParseException;
 import java.util.ArrayList;
@@ -246,7 +248,7 @@ public List<AmoroSnapshotsOfTable> getSnapshots(
 
   @Override
   public List<PartitionFileBaseInfo> getSnapshotDetail(
-      AmoroTable<?> amoroTable, String snapshotId) {
+      AmoroTable<?> amoroTable, String snapshotId, @Nullable String ref) {
     HoodieJavaTable hoodieTable = (HoodieJavaTable) amoroTable.originalTable();
     SyncableFileSystemView fileSystemView = hoodieTable.getHoodieView();
     Map<String, Stream<FileSlice>> ptFsMap =

diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java
@@ -24,7 +24,6 @@
 import org.apache.amoro.TableFormat;
 import org.apache.amoro.api.CommitMetaProducer;
 import org.apache.amoro.process.ProcessStatus;
-import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList;
 import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
 import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
 import org.apache.amoro.shade.guava32.com.google.common.collect.Streams;
@@ -61,7 +60,9 @@
 import org.apache.paimon.manifest.ManifestList;
 import org.apache.paimon.table.DataTable;
 import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.utils.BranchManager;
 import org.apache.paimon.utils.FileStorePathFactory;
+import org.apache.paimon.utils.SnapshotManager;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
@@ -181,9 +182,10 @@ public List<AmoroSnapshotsOfTable> getSnapshots(
     FileStoreTable table = getTable(amoroTable);
     List<AmoroSnapshotsOfTable> snapshotsOfTables = new ArrayList<>();
     Iterator<Snapshot> snapshots;
-    if (PAIMON_MAIN_BRANCH_NAME.equals(ref)) {
+    if (table.branchManager().branchExists(ref) || BranchManager.isMainBranch(ref)) {
+      SnapshotManager snapshotManager = table.snapshotManager().copyWithBranch(ref);
       try {
-        snapshots = table.snapshotManager().snapshots();
+        snapshots = snapshotManager.snapshots();
       } catch (IOException e) {
         throw new RuntimeException(e);
       }
@@ -223,11 +225,17 @@ public List<AmoroSnapshotsOfTable> getSnapshots(
 
   @Override
   public List<PartitionFileBaseInfo> getSnapshotDetail(
-      AmoroTable<?> amoroTable, String snapshotId) {
+      AmoroTable<?> amoroTable, String snapshotId, String ref) {
     FileStoreTable table = getTable(amoroTable);
     List<PartitionFileBaseInfo> amsDataFileInfos = new ArrayList<>();
     long commitId = Long.parseLong(snapshotId);
-    Snapshot snapshot = table.snapshotManager().snapshot(commitId);
+    Snapshot snapshot;
+    if (BranchManager.isMainBranch(ref) || table.branchManager().branchExists(ref)) {
+      snapshot = table.snapshotManager().copyWithBranch(ref).snapshot(commitId);
+    } else {
+      snapshot = table.tagManager().tag(ref);
+    }
+
     FileStore<?> store = table.store();
     FileStorePathFactory fileStorePathFactory = store.pathFactory();
     ManifestList manifestList = store.manifestListFactory().create();
@@ -531,7 +539,14 @@ public List<TagOrBranchInfo> getTableTags(AmoroTable<?> amoroTable) {
 
   @Override
   public List<TagOrBranchInfo> getTableBranches(AmoroTable<?> amoroTable) {
-    return ImmutableList.of(TagOrBranchInfo.MAIN_BRANCH);
+    FileStoreTable table = getTable(amoroTable);
+    List<String> branches = table.branchManager().branches();
+    List<TagOrBranchInfo> branchInfos =
+        branches.stream()
+            .map(name -> new TagOrBranchInfo(name, -1, -1, 0L, 0L, TagOrBranchInfo.BRANCH))
+            .collect(Collectors.toList());
+    branchInfos.add(TagOrBranchInfo.MAIN_BRANCH);
+    return branchInfos;
   }
 
   @Override

diff --git a/amoro-web/src/services/table.service.ts b/amoro-web/src/services/table.service.ts
@@ -127,14 +127,15 @@ export function getDetailBySnapshotId(
     catalog: string
     db: string
     table: string
+    ref: string
     snapshotId: string
     page: number
     pageSize: number
     token?: string
   },
 ) {
-  const { catalog, db, table, snapshotId, page, pageSize, token } = params
-  return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots/${snapshotId}/detail`, { params: { page, pageSize, token } })
+  const { catalog, db, table, snapshotId, page, pageSize, ref, token } = params
+  return request.get(`api/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots/${snapshotId}/detail`, { params: { page, pageSize, ref, token } })
 }
 // get operations
 export function getOperations(

diff --git a/amoro-web/src/views/catalogs/Detail.vue b/amoro-web/src/views/catalogs/Detail.vue
@@ -93,6 +93,11 @@ const isEdit = computed(() => {
 const uploadUrl = computed(() => {
   return '/api/ams/v1/files'
 })
+const uploadHeaders = computed(() => {
+  return {
+    'X-Request-Source': 'Web'
+  };
+});
 const isNewCatalog = computed(() => {
   const catalog = (route.query?.catalogname || '').toString()
   return decodeURIComponent(catalog) === 'new catalog'
@@ -680,6 +685,7 @@ onMounted(() => {
               <a-upload
                 v-if="isEdit" v-model:file-list="config.fileList" name="file" accept=".xml"
                 :show-upload-list="false" :action="uploadUrl" :disabled="config.uploadLoading"
+                :headers="uploadHeaders"
                 @change="(args: UploadChangeParam<UploadFile<any>>) => uploadFile(args, config, 'STORAGE')"
               >
                 <a-button type="primary" ghost :loading="config.uploadLoading" class="g-mr-12">
@@ -729,6 +735,7 @@ onMounted(() => {
                 v-if="isEdit" v-model:file-list="config.fileList" name="file"
                 :accept="config.key === 'auth.kerberos.keytab' ? '.keytab' : '.conf'" :show-upload-list="false"
                 :action="uploadUrl" :disabled="config.uploadLoading"
+                :headers="uploadHeaders"
                 @change="(args: UploadChangeParam<UploadFile<any>>) => uploadFile(args, config)"
               >
                 <a-button type="primary" ghost :loading="config.uploadLoading" class="g-mr-12">

diff --git a/amoro-web/src/views/tables/components/Optimizing.vue b/amoro-web/src/views/tables/components/Optimizing.vue
@@ -30,7 +30,8 @@ const hasBreadcrumb = ref<boolean>(false)
 
 const statusMap = {
   PENDING: { title: 'PENDING', color: '#ffcc00' },
-  ACTIVE: { title: 'ACTIVE', color: '#1890ff' },
+  RUNNING: { title: 'RUNNING', color: '#1890ff' },
+  SUBMITTED: { title: 'SUBMITTED', color: '#4169E1' },
   CLOSED: { title: 'CLOSED', color: '#c9cdd4' },
   SUCCESS: { title: 'SUCCESS', color: '#0ad787' },
   FAILED: { title: 'FAILED', color: '#f5222d' },

diff --git a/amoro-web/src/views/tables/components/Snapshots.vue b/amoro-web/src/views/tables/components/Snapshots.vue
@@ -157,6 +157,7 @@ async function getBreadcrumbTable() {
     const params = {
       ...sourceData,
       snapshotId: snapshotId.value,
+      ref: tblRef.value,
       page: breadcrumbPagination.current,
       pageSize: breadcrumbPagination.pageSize,
     }

diff --git a/dist/pom.xml b/dist/pom.xml
@@ -30,6 +30,10 @@
     <name>Amoro Project AMS Dist</name>
     <url>https://amoro.apache.org</url>
 
+    <properties>
+        <assembly.descriptor.file>src/main/assemblies/bin.xml</assembly.descriptor.file>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.amoro</groupId>
@@ -76,7 +80,7 @@
                         <phase>package</phase>
                         <configuration>
                             <descriptors>
-                                <descriptor>src/main/assemblies/bin.xml</descriptor>
+                                <descriptor>${assembly.descriptor.file}</descriptor>
                             </descriptors>
                             <finalName>apache-amoro-${project.version}-bin</finalName>
                             <appendAssemblyId>false</appendAssemblyId>
@@ -101,4 +105,12 @@
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+            <id>no-plugin-bin</id>
+            <properties>
+                <assembly.descriptor.file>src/main/assemblies/release-bin.xml</assembly.descriptor.file>
+            </properties>
+        </profile>
+    </profiles>
 </project>
diff --git a/dist/src/main/assemblies/release-bin.xml b/dist/src/main/assemblies/release-bin.xml
@@ -0,0 +1,98 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id>bin</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+
+    <includeBaseDirectory>true</includeBaseDirectory>
+    <baseDirectory>amoro-${project.version}</baseDirectory>
+
+    <files>
+        <file>
+            <source>../amoro-ams/target/amoro-ams-${project.version}.jar</source>
+            <outputDirectory>lib/</outputDirectory>
+            <destName>amoro-ams-${project.version}.jar</destName>
+            <fileMode>0644</fileMode>
+        </file>
+        <file>
+            <source>src/main/amoro-bin/conf/env.sh</source>
+            <outputDirectory>conf</outputDirectory>
+            <fileMode>0755</fileMode>
+        </file>
+        <file>
+            <source>../LICENSE-binary</source>
+            <outputDirectory/>
+            <destName>LICENSE</destName>
+            <fileMode>0644</fileMode>
+        </file>
+        <file>
+            <source>../NOTICE-binary</source>
+            <outputDirectory/>
+            <destName>NOTICE</destName>
+            <fileMode>0644</fileMode>
+        </file>
+        <file>
+            <source>../DISCLAIMER</source>
+            <outputDirectory/>
+            <fileMode>0644</fileMode>
+        </file>
+    </files>
+    <fileSets>
+        <fileSet>
+            <directory>../licenses-binary</directory>
+            <outputDirectory>licenses</outputDirectory>
+            <fileMode>0644</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>src/main/amoro-bin/bin</directory>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <lineEnding>unix</lineEnding>
+        </fileSet>
+        <fileSet>
+            <directory>src/main/amoro-bin/conf</directory>
+            <outputDirectory>conf</outputDirectory>
+            <fileMode>0644</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>../amoro-ams/src/main/resources/mysql</directory>
+            <outputDirectory>conf/mysql</outputDirectory>
+            <fileMode>0644</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>../amoro-ams/src/main/resources/postgres</directory>
+            <outputDirectory>conf/postgres</outputDirectory>
+            <fileMode>0644</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>../amoro-ams/src/main/resources/derby</directory>
+            <outputDirectory>conf/derby</outputDirectory>
+            <fileMode>0644</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>../amoro-ams/target/amoro-ams-dependency/lib</directory>
+            <outputDirectory>lib/</outputDirectory>
+            <fileMode>0644</fileMode>
+        </fileSet>
+    </fileSets>
+</assembly>