From 349460cd49c05ea1272b329d4544be94ddeb4827 Mon Sep 17 00:00:00 2001 From: Ikhun Um Date: Fri, 1 Nov 2024 10:54:52 +0900 Subject: [PATCH] Provide a way to trigger Git mirroring manually (#1041) Motivation: - There is no easy way to test Git mirroring configurations. - Some users may want to run Git mirroring manually instead of using the periodic scheduler. Modifications: - Add `POST /projects/{projectName}/mirrors/{mirrorId}/run` REST API to `MirroringServiceV1` so as to trigger a mirroring task. - Add `verboseResponses` to `CentralDogmaConfig` so as to contain full stack traces in error responses. - The cause of mirroring failures will be displayed in the UI for administrators or `verboseResponses` is enabled. - Change `AbstractMirror` to return `MirrorResult` which is used to notify the result in the UI. - Make `MirrorDto.schedule` nullable. - Exclude a mirror configuration from scheduling `schedule` it is null. - Move `MirrorException` to `common` and expose it in the error responses. - Add `cronstrue` dependency to `package.json` to display a human readable description on the mirror form UI. - Increase `maxWidth` of `NotificationWrapper` to render stack traces well. Result: - You can now trigger a mirroring task in the mirror UI. - Fixes #700 - Fixes #236 --- .../client/armeria/ArmeriaCentralDogma.java | 2 + .../centraldogma/common}/MirrorException.java | 8 +- .../internal/api/v1/MirrorDto.java | 8 +- .../it/mirror/git/ForceRefUpdateTest.java | 2 +- .../mirror/git/GitMirrorIntegrationTest.java | 22 ++- .../git/LocalToRemoteGitMirrorTest.java | 26 ++- .../it/mirror/git/MirrorRunnerTest.java | 159 ++++++++++++++++++ .../internal/mirror/AbstractGitMirror.java | 114 ++++++++----- .../internal/mirror/DefaultGitMirror.java | 17 +- .../server/internal/mirror/SshGitMirror.java | 26 ++- .../server/mirror/git/GitMirrorException.java | 46 +++++ .../server/mirror/git/SshMirrorException.java | 34 ++++ .../server/mirror/git/package-info.java | 22 +++ .../DefaultMetaRepositoryWithMirrorTest.java | 1 + ....java => MirrorSchedulingServiceTest.java} | 18 +- .../internal/mirror/MirroringTaskTest.java | 9 +- .../internal/mirror/SshGitMirrorTest.java | 11 +- .../centraldogma/server/CentralDogma.java | 27 ++- .../admin/auth/CsrfTokenAuthorizer.java | 2 + .../admin/auth/SessionTokenAuthorizer.java | 2 + .../internal/api/HttpApiExceptionHandler.java | 5 +- .../server/internal/api/HttpApiUtil.java | 45 +++-- .../internal/api/MirroringServiceV1.java | 30 +++- .../api/auth/ApplicationTokenAuthorizer.java | 7 +- .../internal/mirror/AbstractMirror.java | 75 ++++++--- .../internal/mirror/CentralDogmaMirror.java | 11 +- .../mirror/DefaultMirroringServicePlugin.java | 10 +- .../server/internal/mirror/MirrorRunner.java | 154 +++++++++++++++++ ...vice.java => MirrorSchedulingService.java} | 15 +- .../repository/DefaultMetaRepository.java | 11 +- .../storage/repository/MirrorConfig.java | 19 ++- .../centraldogma/server/mirror/Mirror.java | 6 +- .../server/mirror/MirrorContext.java | 7 +- .../server/mirror/MirrorResult.java | 146 ++++++++++++++++ .../server/mirror/MirrorStatus.java | 31 ++++ .../internal/api/ContentServiceV1Test.java | 6 +- .../server/internal/api/MergeFileTest.java | 12 +- .../internal/api/RepositoryServiceV1Test.java | 6 +- .../internal/api/auth/PermissionTest.java | 2 +- ...=> MirrorSchedulingServicePluginTest.java} | 2 +- webapp/package-lock.json | 10 ++ webapp/package.json | 1 + webapp/src/dogma/common/components/Navbar.tsx | 4 +- .../common/components/NotificationWrapper.tsx | 11 +- webapp/src/dogma/features/api/apiSlice.ts | 9 + webapp/src/dogma/features/auth/Authorized.tsx | 2 +- .../src/dogma/features/mirror/MirrorResult.ts | 25 +++ .../dogma/features/mirror/RunMirrorButton.tsx | 115 +++++++++++++ .../notification/notificationSlice.ts | 22 ++- .../project/settings/mirrors/MirrorDto.ts | 2 +- .../project/settings/mirrors/MirrorForm.tsx | 102 +++++++---- .../project/settings/mirrors/MirrorList.tsx | 42 ++++- .../project/settings/mirrors/MirrorView.tsx | 32 +++- .../features/services/ErrorMessageParser.ts | 6 +- webapp/src/dogma/store.ts | 5 +- .../xds/k8s/v1/XdsKubernetesService.java | 4 +- .../xds/k8s/v1/XdsKubernetesServiceTest.java | 5 +- 57 files changed, 1333 insertions(+), 220 deletions(-) rename {server/src/main/java/com/linecorp/centraldogma/server => common/src/main/java/com/linecorp/centraldogma/common}/MirrorException.java (86%) create mode 100644 it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java create mode 100644 server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/GitMirrorException.java create mode 100644 server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/SshMirrorException.java create mode 100644 server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/package-info.java rename server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/{DefaultMirroringServiceTest.java => MirrorSchedulingServiceTest.java} (81%) create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java rename server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/{DefaultMirroringService.java => MirrorSchedulingService.java} (95%) create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorStatus.java rename server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/{DefaultMirroringServicePluginTest.java => MirrorSchedulingServicePluginTest.java} (98%) create mode 100644 webapp/src/dogma/features/mirror/MirrorResult.ts create mode 100644 webapp/src/dogma/features/mirror/RunMirrorButton.tsx diff --git a/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java b/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java index 72ebd1f99c..26111b34b8 100644 --- a/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java +++ b/client/java-armeria/src/main/java/com/linecorp/centraldogma/internal/client/armeria/ArmeriaCentralDogma.java @@ -89,6 +89,7 @@ import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.MergeQuery; import com.linecorp.centraldogma.common.MergedEntry; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.common.PathPattern; import com.linecorp.centraldogma.common.ProjectExistsException; import com.linecorp.centraldogma.common.ProjectNotFoundException; @@ -133,6 +134,7 @@ public final class ArmeriaCentralDogma extends AbstractCentralDogma { .put(RepositoryExistsException.class.getName(), RepositoryExistsException::new) .put(InvalidPushException.class.getName(), InvalidPushException::new) .put(ReadOnlyException.class.getName(), ReadOnlyException::new) + .put(MirrorException.class.getName(), MirrorException::new) .build(); private final WebClient client; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/MirrorException.java b/common/src/main/java/com/linecorp/centraldogma/common/MirrorException.java similarity index 86% rename from server/src/main/java/com/linecorp/centraldogma/server/MirrorException.java rename to common/src/main/java/com/linecorp/centraldogma/common/MirrorException.java index fcbc164ac1..8cdbbc780a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/MirrorException.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/MirrorException.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2024 LINE Corporation * * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance @@ -14,12 +14,12 @@ * under the License. */ -package com.linecorp.centraldogma.server; +package com.linecorp.centraldogma.common; /** - * A {@link RuntimeException} raised when {@link MirroringService} failed to mirror a repository. + * A {@link CentralDogmaException} raised when failed to mirror a repository. */ -public class MirrorException extends RuntimeException { +public class MirrorException extends CentralDogmaException { private static final long serialVersionUID = 5648624670936197720L; diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java index 92c1eccca7..098a990874 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java @@ -36,6 +36,7 @@ public final class MirrorDto { private final String id; private final boolean enabled; private final String projectName; + @Nullable private final String schedule; private final String direction; private final String localRepo; @@ -52,7 +53,7 @@ public final class MirrorDto { public MirrorDto(@JsonProperty("id") String id, @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("projectName") String projectName, - @JsonProperty("schedule") String schedule, + @JsonProperty("schedule") @Nullable String schedule, @JsonProperty("direction") String direction, @JsonProperty("localRepo") String localRepo, @JsonProperty("localPath") String localPath, @@ -65,7 +66,7 @@ public MirrorDto(@JsonProperty("id") String id, this.id = requireNonNull(id, "id"); this.enabled = firstNonNull(enabled, true); this.projectName = requireNonNull(projectName, "projectName"); - this.schedule = requireNonNull(schedule, "schedule"); + this.schedule = schedule; this.direction = requireNonNull(direction, "direction"); this.localRepo = requireNonNull(localRepo, "localRepo"); this.localPath = requireNonNull(localPath, "localPath"); @@ -92,6 +93,7 @@ public String projectName() { return projectName; } + @Nullable @JsonProperty("schedule") public String schedule() { return schedule; @@ -155,7 +157,7 @@ public boolean equals(Object o) { return id.equals(mirrorDto.id) && enabled == mirrorDto.enabled && projectName.equals(mirrorDto.projectName) && - schedule.equals(mirrorDto.schedule) && + Objects.equals(schedule, mirrorDto.schedule) && direction.equals(mirrorDto.direction) && localRepo.equals(mirrorDto.localRepo) && localPath.equals(mirrorDto.localPath) && diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java index 9d473afa5d..e4cf34e0ca 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java @@ -45,9 +45,9 @@ import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.CentralDogmaBuilder; -import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.internal.mirror.MirrorState; import com.linecorp.centraldogma.server.mirror.MirrorDirection; diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java index 1ee7cced8d..5131b16020 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java @@ -29,6 +29,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.concurrent.CompletionException; import javax.annotation.Nullable; @@ -63,10 +64,11 @@ import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Commit; import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.CentralDogmaBuilder; -import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.internal.JGitUtil; import com.linecorp.centraldogma.server.mirror.MirroringServicePluginConfig; @@ -478,6 +480,23 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N @Nullable String gitignore) { final String localPath0 = localPath == null ? "/" : localPath; final String remoteUri = gitUri + firstNonNull(remotePath, ""); + try { + client.forRepo(projName, Project.REPO_META) + .commit("Add /credentials/none", + Change.ofJsonUpsert("/credentials/none.json", + "{ " + + "\"type\": \"none\", " + + "\"id\": \"none\", " + + "\"enabled\": true " + + '}')) + .push().join(); + } catch (CompletionException e) { + if (e.getCause() instanceof RedundantChangeException) { + // The same content can be pushed several times. + } else { + throw e; + } + } client.forRepo(projName, Project.REPO_META) .commit("Add /mirrors/foo.json", Change.ofJsonUpsert("/mirrors/foo.json", @@ -490,6 +509,7 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"localPath\": \"" + localPath0 + "\"," + " \"remoteUri\": \"" + remoteUri + "\"," + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + + " \"credentialId\": \"none\"," + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + '}')) .push().join(); diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index 2141574a20..aaff0392f1 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletionException; import javax.annotation.Nullable; @@ -53,11 +54,12 @@ import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.CentralDogmaBuilder; -import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.internal.mirror.MirrorState; import com.linecorp.centraldogma.server.mirror.MirrorDirection; @@ -315,7 +317,7 @@ void localToRemote_tooManyBytes() throws Exception { long remainder = MAX_NUM_BYTES + 1; final int defaultFileSize = (int) (MAX_NUM_BYTES / MAX_NUM_FILES * 2); final ArrayList> changes = new ArrayList<>(); - for (int i = 0;; i++) { + for (int i = 0; ; i++) { final int fileSize; if (remainder > defaultFileSize) { remainder -= defaultFileSize; @@ -357,6 +359,23 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N @Nullable String gitignore, MirrorDirection direction) { final String localPath0 = localPath == null ? "/" : localPath; final String remoteUri = gitUri + firstNonNull(remotePath, ""); + try { + client.forRepo(projName, Project.REPO_META) + .commit("Add /credentials/none", + Change.ofJsonUpsert("/credentials/none.json", + "{ " + + "\"type\": \"none\", " + + "\"id\": \"none\", " + + "\"enabled\": true " + + '}')) + .push().join(); + } catch (CompletionException e) { + if (e.getCause() instanceof RedundantChangeException) { + // The same content can be pushed several times. + } else { + throw e; + } + } client.forRepo(projName, Project.REPO_META) .commit("Add /mirrors/foo.json", Change.ofJsonUpsert("/mirrors/foo.json", @@ -369,7 +388,8 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"localPath\": \"" + localPath0 + "\"," + " \"remoteUri\": \"" + remoteUri + "\"," + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + - " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + ',' + + " \"credentialId\": \"none\"" + '}')) .push().join(); } diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java new file mode 100644 index 0000000000..fee0f151c3 --- /dev/null +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java @@ -0,0 +1,159 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.it.mirror.git; + +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.PASSWORD; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.USERNAME; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.getAccessToken; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.common.io.Resources; + +import com.linecorp.armeria.client.BlockingWebClient; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.ResponseEntity; +import com.linecorp.armeria.common.auth.AuthToken; +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.internal.api.v1.PushResultDto; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.internal.credential.PublicKeyCredential; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirrorStatus; +import com.linecorp.centraldogma.testing.internal.auth.TestAuthProviderFactory; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +class MirrorRunnerTest { + + private static final String FOO_PROJ = "foo"; + private static final String BAR_REPO = "bar"; + private static final String PRIVATE_KEY_FILE = "ecdsa_256.openssh"; + private static final String TEST_MIRROR_ID = "test-mirror"; + + @RegisterExtension + static final CentralDogmaExtension dogma = new CentralDogmaExtension() { + + @Override + protected void configure(CentralDogmaBuilder builder) { + builder.authProviderFactory(new TestAuthProviderFactory()); + builder.administrators(USERNAME); + } + + @Override + protected void scaffold(CentralDogma client) { + client.createProject(FOO_PROJ).join(); + client.createRepository(FOO_PROJ, BAR_REPO).join(); + } + }; + + private BlockingWebClient adminClient; + + @BeforeEach + void setUp() throws Exception { + final String adminToken = getAccessToken(dogma.httpClient(), USERNAME, PASSWORD); + adminClient = WebClient.builder(dogma.httpClient().uri()) + .auth(AuthToken.ofOAuth2(adminToken)) + .build() + .blocking(); + } + + @Test + void triggerMirroring() throws Exception { + final PublicKeyCredential credential = getCredential(); + ResponseEntity response = + adminClient.prepare() + .post("/api/v1/projects/{proj}/credentials") + .pathParam("proj", FOO_PROJ) + .contentJson(credential) + .asJson(PushResultDto.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.CREATED); + + final MirrorDto newMirror = newMirror(); + response = adminClient.prepare() + .post("/api/v1/projects/{proj}/mirrors") + .pathParam("proj", FOO_PROJ) + .contentJson(newMirror) + .asJson(PushResultDto.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.CREATED); + + for (int i = 0; i < 3; i++) { + final ResponseEntity mirrorResponse = + adminClient.prepare() + .post("/api/v1/projects/{proj}/mirrors/{mirrorId}/run") + .pathParam("proj", FOO_PROJ) + .pathParam("mirrorId", TEST_MIRROR_ID) + .asJson(MirrorResult.class) + .execute(); + + assertThat(mirrorResponse.status()).isEqualTo(HttpStatus.OK); + if (i == 0) { + assertThat(mirrorResponse.content().mirrorStatus()).isEqualTo(MirrorStatus.SUCCESS); + assertThat(mirrorResponse.content().description()) + .contains("'git+ssh://github.com/line/centraldogma-authtest.git#main' to " + + "the repository 'bar', revision: 2"); + } else { + assertThat(mirrorResponse.content().mirrorStatus()).isEqualTo(MirrorStatus.UP_TO_DATE); + assertThat(mirrorResponse.content().description()) + .contains("Repository 'foo/bar' already at"); + } + } + } + + private static MirrorDto newMirror() { + return new MirrorDto(TEST_MIRROR_ID, + true, + FOO_PROJ, + null, + "REMOTE_TO_LOCAL", + BAR_REPO, + "/", + "git+ssh", + "github.com/line/centraldogma-authtest.git", + "/", + "main", + null, + PRIVATE_KEY_FILE); + } + + private static PublicKeyCredential getCredential() throws Exception { + final String publicKeyFile = "ecdsa_256.openssh.pub"; + + final byte[] privateKeyBytes = + Resources.toByteArray(GitMirrorAuthTest.class.getResource(PRIVATE_KEY_FILE)); + final byte[] publicKeyBytes = + Resources.toByteArray(GitMirrorAuthTest.class.getResource(publicKeyFile)); + final String privateKey = new String(privateKeyBytes, StandardCharsets.UTF_8).trim(); + final String publicKey = new String(publicKeyBytes, StandardCharsets.UTF_8).trim(); + + return new PublicKeyCredential( + PRIVATE_KEY_FILE, + true, + "git", + publicKey, + privateKey, + null); + } +} diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java index 19b84e04b8..225339517a 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java @@ -24,12 +24,14 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.time.Instant; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletionException; import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -80,14 +82,19 @@ import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.EntryType; import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.MirrorException; +import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.internal.Util; -import com.linecorp.centraldogma.server.MirrorException; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.credential.Credential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirrorStatus; +import com.linecorp.centraldogma.server.mirror.git.GitMirrorException; import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -115,7 +122,7 @@ abstract class AbstractGitMirror extends AbstractMirror { @Nullable private IgnoreNode ignoreNode; - AbstractGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + AbstractGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { @@ -169,14 +176,16 @@ GitWithAuth openGit(File workDir, } } - void mirrorLocalToRemote( - GitWithAuth git, int maxNumFiles, long maxNumBytes) throws GitAPIException, IOException { + MirrorResult mirrorLocalToRemote( + GitWithAuth git, int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws GitAPIException, IOException { // TODO(minwoox): Early return if the remote does not have any updates. final Ref headBranchRef = getHeadBranchRef(git); final String headBranchRefName = headBranchRef.getName(); final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRefName); final org.eclipse.jgit.lib.Repository gitRepository = git.getRepository(); + final String description; try (ObjectReader reader = gitRepository.newObjectReader(); TreeWalk treeWalk = new TreeWalk(reader); RevWalk revWalk = new RevWalk(reader)) { @@ -190,9 +199,12 @@ void mirrorLocalToRemote( final Revision remoteCurrentRevision = remoteCurrentRevision(reader, treeWalk, mirrorStatePath); if (localHead.equals(remoteCurrentRevision)) { // The remote repository is up-to date. - logger.debug("The remote repository '{}#{}' already at {}. Local repository: '{}'", - remoteRepoUri(), remoteBranch(), localHead, localRepo().name()); - return; + description = String.format( + "The remote repository '%s#%s' already at %s. Local repository: '%s/%s'", + remoteRepoUri(), remoteBranch(), localHead, + localRepo().parent().name(), localRepo().name()); + logger.debug(description); + return newMirrorResult(MirrorStatus.UP_TO_DATE, description, triggeredTime); } // Reset to traverse the tree from the first. @@ -217,8 +229,12 @@ dirCache, new InsertText(mirrorStatePath.substring(1), // Strip the leading '/'. Jackson.writeValueAsPrettyString(mirrorState) + '\n')); } + final String summary = "Mirror '" + localRepo().name() + "' at " + localHead + + " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n"; + description = summary; final ObjectId nextCommitId = - commit(gitRepository, dirCache, headCommitId, localHead); + commit(gitRepository, dirCache, headCommitId, summary); + logger.info(summary); updateRef(gitRepository, revWalk, headBranchRefName, nextCommitId); } @@ -227,24 +243,37 @@ dirCache, new InsertText(mirrorStatePath.substring(1), // Strip the leading '/'. .setAtomic(true) .setTimeout(GIT_TIMEOUT_SECS) .call(); + return newMirrorResult(MirrorStatus.SUCCESS, description, triggeredTime); } - void mirrorRemoteToLocal( - GitWithAuth git, CommandExecutor executor, int maxNumFiles, long maxNumBytes) throws Exception { + MirrorResult mirrorRemoteToLocal( + GitWithAuth git, CommandExecutor executor, int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws Exception { final String summary; final String detail; + final Ref headBranchRef; + final ObjectId headCommitId; final Map> changes = new HashMap<>(); - final Ref headBranchRef = getHeadBranchRef(git); - - final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME; final Revision localRev = localRepo().normalizeNow(Revision.HEAD); - if (!needsFetch(headBranchRef, mirrorStatePath, localRev)) { - return; + final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME; + try { + headBranchRef = getHeadBranchRef(git); + if (!needsFetch(headBranchRef, mirrorStatePath, localRev)) { + return newMirrorResultForUpToDate(headBranchRef, triggeredTime); + } + + // Update the head commit ID again because there's a chance a commit is pushed between the + // getHeadBranchRefName and fetchRemoteHeadAndGetCommitId calls. + headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRef.getName()); + } catch (Exception e) { + String message = "Failed to fetch the remote repository '" + git.remoteUri() + + "' to the local repository '" + localPath() + "'."; + if (e.getMessage() != null) { + message += " (reason: " + e.getMessage(); + } + throw new GitMirrorException(message, e); } - // Update the head commit ID again because there's a chance a commit is pushed between the - // getHeadBranchRefName and fetchRemoteHeadAndGetCommitId calls. - final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRef.getName()); try (ObjectReader reader = git.getRepository().newObjectReader(); TreeWalk treeWalk = new TreeWalk(reader); RevWalk revWalk = new RevWalk(reader)) { @@ -301,14 +330,12 @@ void mirrorRemoteToLocal( if (++numFiles > maxNumFiles) { throwMirrorException(maxNumFiles, "files"); - return; } final ObjectId objectId = treeWalk.getObjectId(0); final long contentLength = reader.getObjectSize(objectId, ObjectReader.OBJ_ANY); if (numBytes > maxNumBytes - contentLength) { throwMirrorException(maxNumBytes, "bytes"); - return; } numBytes += contentLength; @@ -337,9 +364,28 @@ void mirrorRemoteToLocal( } }); - executor.execute(Command.push( - MIRROR_AUTHOR, localRepo().parent().name(), localRepo().name(), - Revision.HEAD, summary, detail, Markup.PLAINTEXT, changes.values())).join(); + try { + final CommitResult commitResult = executor.execute(Command.push( + MIRROR_AUTHOR, localRepo().parent().name(), localRepo().name(), + Revision.HEAD, summary, detail, Markup.PLAINTEXT, changes.values())).join(); + final String description = summary + ", revision: " + commitResult.revision().text(); + return newMirrorResult(MirrorStatus.SUCCESS, description, triggeredTime); + } catch (CompletionException e) { + if (e.getCause() instanceof RedundantChangeException) { + return newMirrorResultForUpToDate(headBranchRef, triggeredTime); + } + throw e; + } + } + + private MirrorResult newMirrorResultForUpToDate(Ref headBranchRef, Instant triggeredTime) { + final String abbrId = headBranchRef.getObjectId().abbreviate(OBJECT_ID_ABBREV_STRING_LENGTH).name(); + final String message = String.format("Repository '%s/%s' already at %s, %s#%s", + localRepo().parent().name(), localRepo().name(), abbrId, + remoteRepoUri(), remoteBranch()); + // The local repository is up-to date. + logger.debug(message); + return newMirrorResult(MirrorStatus.UP_TO_DATE, message, triggeredTime); } private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision localRev) @@ -354,13 +400,7 @@ private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision l } final ObjectId headCommitId = headBranchRef.getObjectId(); - if (headCommitId.name().equals(localSourceRevision)) { - final String abbrId = headCommitId.abbreviate(OBJECT_ID_ABBREV_STRING_LENGTH).name(); - logger.info("Repository '{}' already at {}, {}#{}", localRepo().name(), abbrId, - remoteRepoUri(), remoteBranch()); - return false; - } - return true; + return !headCommitId.name().equals(localSourceRevision); } private Ref getHeadBranchRef(GitWithAuth git) throws GitAPIException { @@ -405,8 +445,8 @@ private static Ref findHeadBranchRef(GitWithAuth git, String headBranchRefName, if (headBranchRef.isPresent()) { return headBranchRef.get(); } - throw new MirrorException("Remote does not have " + headBranchRefName + " branch. remote: " + - git.remoteUri()); + throw new GitMirrorException("Remote does not have " + headBranchRefName + " branch. remote: " + + git.remoteUri()); } private static String generateCommitDetail(RevCommit headCommit) { @@ -674,8 +714,8 @@ private static String sanitizeText(String text) { return text; } - private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache, - ObjectId headCommitId, Revision localHead) throws IOException { + private static ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache, + ObjectId headCommitId, String message) throws IOException { try (ObjectInserter inserter = gitRepository.newObjectInserter()) { // flush the current index to repository and get the result tree object id. final ObjectId nextTreeId = dirCache.writeTree(inserter); @@ -691,11 +731,7 @@ private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache commitBuilder.setTreeId(nextTreeId); commitBuilder.setEncoding(UTF_8); commitBuilder.setParentId(headCommitId); - - final String summary = "Mirror '" + localRepo().name() + "' at " + localHead + - " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n"; - logger.info(summary); - commitBuilder.setMessage(summary); + commitBuilder.setMessage(message); final ObjectId nextCommitId = inserter.insert(commitBuilder); inserter.flush(); diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java index 069fe1375d..6687d3cc9c 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java @@ -21,6 +21,7 @@ import java.io.File; import java.net.URI; +import java.time.Instant; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -36,13 +37,14 @@ import com.linecorp.centraldogma.server.internal.credential.AccessTokenCredential; import com.linecorp.centraldogma.server.internal.credential.PasswordCredential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; import com.linecorp.centraldogma.server.storage.repository.Repository; final class DefaultGitMirror extends AbstractGitMirror { private static final Consumer> NOOP_CONFIGURATOR = command -> {}; - DefaultGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + DefaultGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { @@ -51,9 +53,11 @@ final class DefaultGitMirror extends AbstractGitMirror { } @Override - protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes, + Instant triggeredTime) + throws Exception { try (GitWithAuth git = openGit(workDir, transportCommandConfigurator())) { - mirrorLocalToRemote(git, maxNumFiles, maxNumBytes); + return mirrorLocalToRemote(git, maxNumFiles, maxNumBytes, triggeredTime); } } @@ -78,10 +82,11 @@ protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumByt } @Override - protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, - int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor, + int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws Exception { try (GitWithAuth git = openGit(workDir, transportCommandConfigurator())) { - mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes); + return mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes, triggeredTime); } } diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java index 690fd51f6b..d842d255a7 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java @@ -24,6 +24,7 @@ import java.net.URISyntaxException; import java.security.GeneralSecurityException; import java.security.KeyPair; +import java.time.Instant; import java.util.Collection; import javax.annotation.Nullable; @@ -55,12 +56,14 @@ import com.cronutils.model.Cron; import com.google.common.annotations.VisibleForTesting; -import com.linecorp.centraldogma.server.MirrorException; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.credential.Credential; import com.linecorp.centraldogma.server.internal.credential.PasswordCredential; import com.linecorp.centraldogma.server.internal.credential.PublicKeyCredential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.git.SshMirrorException; import com.linecorp.centraldogma.server.storage.repository.Repository; final class SshGitMirror extends AbstractGitMirror { @@ -79,7 +82,7 @@ final class SshGitMirror extends AbstractGitMirror { // We might create multiple BouncyCastleRandom later and poll them, if necessary. private static final BouncyCastleRandom bounceCastleRandom = new BouncyCastleRandom(); - SshGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + SshGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { @@ -89,28 +92,31 @@ final class SshGitMirror extends AbstractGitMirror { } @Override - protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes, + Instant triggeredTime) + throws Exception { final URIish remoteUri = remoteUri(); try (SshClient sshClient = createSshClient(); ClientSession session = createSession(sshClient, remoteUri)) { final DefaultGitSshdSessionFactory sessionFactory = new DefaultGitSshdSessionFactory(sshClient, session); try (GitWithAuth git = openGit(workDir, remoteUri, sessionFactory::configureCommand)) { - mirrorLocalToRemote(git, maxNumFiles, maxNumBytes); + return mirrorLocalToRemote(git, maxNumFiles, maxNumBytes, triggeredTime); } } } @Override - protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, - int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor, + int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws Exception { final URIish remoteUri = remoteUri(); try (SshClient sshClient = createSshClient(); ClientSession session = createSession(sshClient, remoteUri)) { final DefaultGitSshdSessionFactory sessionFactory = new DefaultGitSshdSessionFactory(sshClient, session); try (GitWithAuth git = openGit(workDir, remoteUri, sessionFactory::configureCommand)) { - mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes); + return mirrorRemoteToLocal(git, executor, maxNumFiles, maxNumBytes, triggeredTime); } } } @@ -176,7 +182,11 @@ static ClientSession createSession(SshClient sshClient, URIish uri) { if (session != null) { session.close(true); } - throw new RuntimeException("failed to create a session for " + uri + " from " + sshClient, t); + String message = "Failed to create a session for '" + uri + "'."; + if (t.getMessage() != null) { + message += " (reason: " + t.getMessage() + ')'; + } + throw new SshMirrorException(message, t); } } diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/GitMirrorException.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/GitMirrorException.java new file mode 100644 index 0000000000..e8ac48fff0 --- /dev/null +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/GitMirrorException.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.mirror.git; + +import com.linecorp.centraldogma.common.MirrorException; + +/** + * A {@link MirrorException} raised when failed to mirror a Git repository. + */ +public class GitMirrorException extends MirrorException { + + private static final long serialVersionUID = 4510614751276168395L; + + /** + * Creates a new instance. + */ + public GitMirrorException() {} + + /** + * Creates a new instance. + */ + public GitMirrorException(String message) { + super(message); + } + + /** + * Creates a new instance. + */ + public GitMirrorException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/SshMirrorException.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/SshMirrorException.java new file mode 100644 index 0000000000..24d1697e17 --- /dev/null +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/SshMirrorException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.mirror.git; + +import com.linecorp.centraldogma.common.MirrorException; + +/** + * A {@link MirrorException} raised when failed to create an SSH session. + */ +public final class SshMirrorException extends MirrorException { + + private static final long serialVersionUID = -8887732097679008838L; + + /** + * Creates a new instance. + */ + public SshMirrorException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/package-info.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/package-info.java new file mode 100644 index 0000000000..930737144f --- /dev/null +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/mirror/git/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright 2021 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +/** + * Central Dogma Git mirroring. + */ +@NonNullByDefault +package com.linecorp.centraldogma.server.mirror.git; + +import com.linecorp.centraldogma.common.util.NonNullByDefault; diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java index 3dcd58cbc7..bdb586ae6d 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java @@ -148,6 +148,7 @@ void testMirror(boolean useRawApi) { '{' + " \"id\": \"foo\"," + " \"enabled\": true," + + " \"schedule\": \"" + DEFAULT_SCHEDULE + "\"," + " \"direction\": \"LOCAL_TO_REMOTE\"," + " \"localRepo\": \"foo\"," + " \"localPath\": \"/mirrors/foo\"," + diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java similarity index 81% rename from server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java rename to server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java index 8751a2704e..05423c90c1 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java @@ -22,6 +22,7 @@ import java.io.File; import java.net.URI; +import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; @@ -39,6 +40,8 @@ import com.linecorp.centraldogma.server.credential.Credential; import com.linecorp.centraldogma.server.mirror.Mirror; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirrorStatus; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.server.storage.repository.MetaRepository; @@ -47,7 +50,7 @@ import io.micrometer.core.instrument.simple.SimpleMeterRegistry; -class DefaultMirroringServiceTest { +class MirrorSchedulingServiceTest { @TempDir static File temporaryFolder; @@ -74,20 +77,25 @@ void mirroringTaskShouldNeverBeRejected() { Credential.FALLBACK, r, "/", URI.create("unused://uri"), "/", "", null) { @Override - protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) {} + protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes, + Instant triggeredTime) { + return newMirrorResult(MirrorStatus.UP_TO_DATE, null, Instant.now()); + } @Override - protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, - int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor, + int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws Exception { // Sleep longer than mirroring interval so that the workers fall behind. taskCounter.incrementAndGet(); Thread.sleep(2000); + return newMirrorResult(MirrorStatus.SUCCESS, null, Instant.now()); } }; when(mr.mirrors()).thenReturn(CompletableFuture.completedFuture(ImmutableList.of(mirror))); - final DefaultMirroringService service = new DefaultMirroringService( + final MirrorSchedulingService service = new MirrorSchedulingService( temporaryFolder, pm, new SimpleMeterRegistry(), 1, 1, 1); final CommandExecutor executor = mock(CommandExecutor.class); service.start(executor); diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java index 51249cfd6e..67fa23069c 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java @@ -25,14 +25,18 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; +import java.time.Instant; + import org.junit.jupiter.api.Test; import com.linecorp.armeria.common.metric.MoreMeters; import com.linecorp.centraldogma.server.mirror.Mirror; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirrorStatus; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -44,7 +48,8 @@ void testSuccessMetrics() { final MeterRegistry meterRegistry = new SimpleMeterRegistry(); Mirror mirror = newMirror("git://a.com/b.git", DefaultGitMirror.class, "foo", "bar"); mirror = spy(mirror); - doNothing().when(mirror).mirror(any(), any(), anyInt(), anyLong()); + doReturn(new MirrorResult(mirror.id(), "foo", "bar", MirrorStatus.SUCCESS, "", Instant.now())) + .when(mirror).mirror(any(), any(), anyInt(), anyLong()); new MirroringTask(mirror, "foo", meterRegistry).run(null, null, 0, 0L); assertThat(MoreMeters.measureAll(meterRegistry)) .contains(entry("mirroring.result#count{direction=LOCAL_TO_REMOTE,localPath=/," + diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirrorTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirrorTest.java index f530fa9cc6..3940102e92 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirrorTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirrorTest.java @@ -32,6 +32,8 @@ import org.eclipse.jgit.transport.URIish; import org.junit.jupiter.api.Test; +import com.linecorp.centraldogma.server.mirror.git.SshMirrorException; + class SshGitMirrorTest { @Test @@ -46,9 +48,12 @@ public void sessionEstablished(Session session) { }); client.start(); - assertThatThrownBy(() -> - createSession(client, new URIish("https://github.com/line/centraldogma-authtest.git")) - ).isExactlyInstanceOf(RuntimeException.class); + assertThatThrownBy(() -> { + createSession(client, new URIish( + "https://github.com/line/centraldogma-authtest.git")); + }).isExactlyInstanceOf(SshMirrorException.class) + .hasMessage("Failed to create a session for 'https://github.com/line/centraldogma-authtest.git'." + + " (reason: No username specified when the session was created)"); final Session session = sessionRef.get(); assertThat(session).isNotNull(); assertThat(session.isClosed()).isTrue(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java index f7b2fd0b35..99cbd72aaa 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java @@ -148,6 +148,7 @@ import com.linecorp.centraldogma.server.internal.api.auth.RequiresRoleDecorator.RequiresRoleDecoratorFactory; import com.linecorp.centraldogma.server.internal.api.converter.HttpApiRequestConverter; import com.linecorp.centraldogma.server.internal.mirror.DefaultMirroringServicePlugin; +import com.linecorp.centraldogma.server.internal.mirror.MirrorRunner; import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor; import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; @@ -250,6 +251,8 @@ public static CentralDogma forConfig(File configFile) throws IOException { private ServerStatusManager statusManager; @Nullable private InternalProjectInitializer projectInitializer; + @Nullable + private volatile MirrorRunner mirrorRunner; CentralDogma(CentralDogmaConfig cfg, MeterRegistry meterRegistry) { this.cfg = requireNonNull(cfg, "cfg"); @@ -438,7 +441,7 @@ private void doStart() throws Exception { this.server = server; this.sessionManager = sessionManager; } else { - doStop(server, executor, pm, repositoryWorker, purgeWorker, sessionManager); + doStop(server, executor, pm, repositoryWorker, purgeWorker, sessionManager, mirrorRunner); } } } @@ -567,7 +570,6 @@ private Server startServer(ProjectManager pm, CommandExecutor executor, @Nullable SessionManager sessionManager, InternalProjectInitializer projectInitializer) { final ServerBuilder sb = Server.builder(); - sb.verboseResponses(true); cfg.ports().forEach(sb::port); final boolean needsTls = @@ -814,7 +816,9 @@ private void configureHttpApi(ServerBuilder sb, .annotatedService(new CredentialServiceV1(projectApiManager, executor)); if (GIT_MIRROR_ENABLED) { - apiV1ServiceBuilder.annotatedService(new MirroringServiceV1(projectApiManager, executor)); + mirrorRunner = new MirrorRunner(projectApiManager, executor, cfg, meterRegistry); + apiV1ServiceBuilder.annotatedService(new MirroringServiceV1(projectApiManager, executor, + mirrorRunner)); } apiV1ServiceBuilder.annotatedService() @@ -1021,12 +1025,14 @@ private void doStop() { final ExecutorService repositoryWorker = this.repositoryWorker; final ExecutorService purgeWorker = this.purgeWorker; final SessionManager sessionManager = this.sessionManager; + final MirrorRunner mirrorRunner = this.mirrorRunner; this.server = null; this.executor = null; this.pm = null; this.repositoryWorker = null; this.sessionManager = null; + this.mirrorRunner = null; projectInitializer = null; if (meterRegistryToBeClosed != null) { assert meterRegistry instanceof CompositeMeterRegistry; @@ -1036,7 +1042,7 @@ private void doStop() { } logger.info("Stopping the Central Dogma .."); - if (!doStop(server, executor, pm, repositoryWorker, purgeWorker, sessionManager)) { + if (!doStop(server, executor, pm, repositoryWorker, purgeWorker, sessionManager, mirrorRunner)) { logger.warn("Stopped the Central Dogma with failure."); } else { logger.info("Stopped the Central Dogma successfully."); @@ -1047,7 +1053,7 @@ private static boolean doStop( @Nullable Server server, @Nullable CommandExecutor executor, @Nullable ProjectManager pm, @Nullable ExecutorService repositoryWorker, @Nullable ExecutorService purgeWorker, - @Nullable SessionManager sessionManager) { + @Nullable SessionManager sessionManager, @Nullable MirrorRunner mirrorRunner) { boolean success = true; try { @@ -1116,6 +1122,17 @@ private static boolean doStop( success = false; } + try { + if (mirrorRunner != null) { + logger.info("Stopping the mirror runner.."); + mirrorRunner.close(); + logger.info("Stopped the mirror runner."); + } + } catch (Throwable t) { + success = false; + logger.warn("Failed to stop the mirror runner:", t); + } + try { if (server != null) { logger.info("Stopping the RPC server .."); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java index dc3c7125b0..fac3340046 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java @@ -26,6 +26,7 @@ import com.linecorp.armeria.server.auth.Authorizer; import com.linecorp.armeria.server.thrift.THttpService; import com.linecorp.centraldogma.internal.CsrfToken; +import com.linecorp.centraldogma.server.internal.api.HttpApiUtil; import com.linecorp.centraldogma.server.metadata.User; /** @@ -39,6 +40,7 @@ public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest final OAuth2Token token = AuthTokenExtractors.oAuth2().apply(data.headers()); if (token != null && CsrfToken.ANONYMOUS.equals(token.accessToken())) { AuthUtil.setCurrentUser(ctx, User.ADMIN); + HttpApiUtil.setVerboseResponses(ctx, User.ADMIN); return CompletableFuture.completedFuture(true); } else { return CompletableFuture.completedFuture(false); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java index f151680f26..e03094da85 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java @@ -31,6 +31,7 @@ import com.linecorp.armeria.server.auth.AuthTokenExtractors; import com.linecorp.armeria.server.auth.Authorizer; import com.linecorp.centraldogma.server.auth.SessionManager; +import com.linecorp.centraldogma.server.internal.api.HttpApiUtil; import com.linecorp.centraldogma.server.metadata.User; /** @@ -64,6 +65,7 @@ public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest final User user = new User(username, roles); ctx.logBuilder().authenticatedUser("user/" + username); AuthUtil.setCurrentUser(ctx, user); + HttpApiUtil.setVerboseResponses(ctx, user); return true; }); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java index 5528e14f57..10fc1172d4 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java @@ -36,6 +36,7 @@ import com.linecorp.centraldogma.common.EntryNoContentException; import com.linecorp.centraldogma.common.EntryNotFoundException; import com.linecorp.centraldogma.common.InvalidPushException; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.common.ProjectExistsException; import com.linecorp.centraldogma.common.ProjectNotFoundException; import com.linecorp.centraldogma.common.QueryExecutionException; @@ -109,7 +110,9 @@ public final class HttpApiExceptionHandler implements ServerErrorHandler { .put(InvalidPushException.class, (ctx, cause) -> newResponse(ctx, HttpStatus.BAD_REQUEST, cause)) .put(ReadOnlyException.class, - (ctx, cause) -> newResponse(ctx, HttpStatus.SERVICE_UNAVAILABLE, cause)); + (ctx, cause) -> newResponse(ctx, HttpStatus.SERVICE_UNAVAILABLE, cause)) + .put(MirrorException.class, + (ctx, cause) -> newResponse(ctx, HttpStatus.INTERNAL_SERVER_ERROR, cause)); exceptionHandlers = builder.build(); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java index 4f5d90b6f3..a4939bb2d0 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.internal.api; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.nullToEmpty; import static java.util.Objects.requireNonNull; @@ -38,15 +39,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.linecorp.armeria.common.Flags; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; -import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.logging.LogLevel; import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.server.HttpResponseException; +import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.centraldogma.common.ShuttingDownException; import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.metadata.User; + +import io.netty.util.AttributeKey; /** * A utility class which provides common functions for HTTP API. @@ -54,6 +59,9 @@ //TODO(minwoox) change this class to package-local when the admin API is integrated with HTTP API public final class HttpApiUtil { + private static final AttributeKey VERBOSE_RESPONSES = + AttributeKey.valueOf(HttpApiUtil.class, "VERBOSE_RESPONSES"); + private static final Logger logger = LoggerFactory.getLogger(HttpApiUtil.class); private static final String ERROR_MESSAGE_FORMAT = "{} Returning a {} response: {}"; @@ -67,7 +75,7 @@ public final class HttpApiUtil { * Throws a newly created {@link HttpResponseException} with the specified {@link HttpStatus} and * {@code message}. */ - public static T throwResponse(RequestContext ctx, HttpStatus status, String message) { + public static T throwResponse(ServiceRequestContext ctx, HttpStatus status, String message) { throw HttpResponseException.of(newResponse(ctx, status, message)); } @@ -75,7 +83,8 @@ public static T throwResponse(RequestContext ctx, HttpStatus status, String * Throws a newly created {@link HttpResponseException} with the specified {@link HttpStatus} and * the formatted message. */ - public static T throwResponse(RequestContext ctx, HttpStatus status, String format, Object... args) { + public static T throwResponse(ServiceRequestContext ctx, HttpStatus status, String format, + Object... args) { throw HttpResponseException.of(newResponse(ctx, status, format, args)); } @@ -83,7 +92,8 @@ public static T throwResponse(RequestContext ctx, HttpStatus status, String * Throws a newly created {@link HttpResponseException} with the specified {@link HttpStatus}, * {@code cause} and {@code message}. */ - public static T throwResponse(RequestContext ctx, HttpStatus status, Throwable cause, String message) { + public static T throwResponse(ServiceRequestContext ctx, HttpStatus status, Throwable cause, + String message) { throw HttpResponseException.of(newResponse(ctx, status, cause, message)); } @@ -91,7 +101,7 @@ public static T throwResponse(RequestContext ctx, HttpStatus status, Throwab * Throws a newly created {@link HttpResponseException} with the specified {@link HttpStatus}, * {@code cause} and the formatted message. */ - public static T throwResponse(RequestContext ctx, HttpStatus status, Throwable cause, + public static T throwResponse(ServiceRequestContext ctx, HttpStatus status, Throwable cause, String format, Object... args) { throw HttpResponseException.of(newResponse(ctx, status, cause, format, args)); } @@ -100,7 +110,7 @@ public static T throwResponse(RequestContext ctx, HttpStatus status, Throwab * Returns a newly created {@link HttpResponse} with the specified {@link HttpStatus} and the formatted * message. */ - public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, + public static HttpResponse newResponse(ServiceRequestContext ctx, HttpStatus status, String format, Object... args) { requireNonNull(ctx, "ctx"); requireNonNull(status, "status"); @@ -112,7 +122,7 @@ public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, /** * Returns a newly created {@link HttpResponse} with the specified {@link HttpStatus} and {@code message}. */ - public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, String message) { + public static HttpResponse newResponse(ServiceRequestContext ctx, HttpStatus status, String message) { requireNonNull(ctx, "ctx"); requireNonNull(status, "status"); requireNonNull(message, "message"); @@ -122,7 +132,7 @@ public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, St /** * Returns a newly created {@link HttpResponse} with the specified {@link HttpStatus} and {@code cause}. */ - public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, Throwable cause) { + public static HttpResponse newResponse(ServiceRequestContext ctx, HttpStatus status, Throwable cause) { requireNonNull(ctx, "ctx"); requireNonNull(status, "status"); requireNonNull(cause, "cause"); @@ -133,7 +143,7 @@ public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, Th * Returns a newly created {@link HttpResponse} with the specified {@link HttpStatus}, {@code cause} and * the formatted message. */ - public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, Throwable cause, + public static HttpResponse newResponse(ServiceRequestContext ctx, HttpStatus status, Throwable cause, String format, Object... args) { requireNonNull(ctx, "ctx"); requireNonNull(status, "status"); @@ -148,7 +158,7 @@ public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, Th * Returns a newly created {@link HttpResponse} with the specified {@link HttpStatus}, {@code cause} and * {@code message}. */ - public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, + public static HttpResponse newResponse(ServiceRequestContext ctx, HttpStatus status, Throwable cause, String message) { requireNonNull(ctx, "ctx"); requireNonNull(status, "status"); @@ -158,7 +168,7 @@ public static HttpResponse newResponse(RequestContext ctx, HttpStatus status, return newResponse0(ctx, status, cause, message); } - private static HttpResponse newResponse0(RequestContext ctx, HttpStatus status, + private static HttpResponse newResponse0(ServiceRequestContext ctx, HttpStatus status, @Nullable Throwable cause, @Nullable String message) { checkArgument(!status.isContentAlwaysEmpty(), "status: %s (expected: a status with non-empty content)", status); @@ -175,6 +185,10 @@ private static HttpResponse newResponse0(RequestContext ctx, HttpStatus status, final String m = nullToEmpty(message); node.put("message", m); + if (cause != null && isVerboseResponse(ctx)) { + node.put("detail", Exceptions.traceText(cause)); + } + final LogLevel logLevel; switch (status.codeClass()) { case SERVER_ERROR: @@ -294,5 +308,14 @@ public static void throwUnsafelyIfNonNull(@Nullable Throwable cause) { }; } + public static void setVerboseResponses(ServiceRequestContext ctx, User user) { + ctx.setAttr(VERBOSE_RESPONSES, Flags.verboseResponses() || user.isAdmin()); + } + + private static boolean isVerboseResponse(ServiceRequestContext ctx) { + final Boolean verboseResponses = ctx.attr(VERBOSE_RESPONSES); + return firstNonNull(verboseResponses, false); + } + private HttpApiUtil() {} } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java index a92c591a97..2271e1e62c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java @@ -22,6 +22,9 @@ import java.net.URI; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import com.cronutils.model.Cron; import com.linecorp.armeria.server.annotation.ConsumesJson; import com.linecorp.armeria.server.annotation.Get; @@ -30,14 +33,17 @@ import com.linecorp.armeria.server.annotation.ProducesJson; import com.linecorp.armeria.server.annotation.Put; import com.linecorp.armeria.server.annotation.StatusCode; +import com.linecorp.armeria.server.annotation.decorator.RequestTimeout; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.internal.api.v1.MirrorDto; import com.linecorp.centraldogma.internal.api.v1.PushResultDto; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.internal.api.auth.RequiresReadPermission; import com.linecorp.centraldogma.server.internal.api.auth.RequiresWritePermission; +import com.linecorp.centraldogma.server.internal.mirror.MirrorRunner; import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; import com.linecorp.centraldogma.server.mirror.Mirror; +import com.linecorp.centraldogma.server.mirror.MirrorResult; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.MetaRepository; @@ -52,10 +58,13 @@ public class MirroringServiceV1 extends AbstractService { // - Add Java APIs to the CentralDogma client private final ProjectApiManager projectApiManager; + private final MirrorRunner mirrorRunner; - public MirroringServiceV1(ProjectApiManager projectApiManager, CommandExecutor executor) { + public MirroringServiceV1(ProjectApiManager projectApiManager, CommandExecutor executor, + MirrorRunner mirrorRunner) { super(executor); this.projectApiManager = projectApiManager; + this.mirrorRunner = mirrorRunner; } /** @@ -81,7 +90,6 @@ public CompletableFuture> listMirrors(@Param String projectName) @RequiresReadPermission(repository = Project.REPO_META) @Get("/projects/{projectName}/mirrors/{id}") public CompletableFuture getMirror(@Param String projectName, @Param String id) { - return metaRepo(projectName).mirror(id).thenApply(mirror -> { return convertToMirrorDto(projectName, mirror); }); @@ -125,11 +133,27 @@ private CompletableFuture createOrUpdate(String projectName, }); } + /** + * POST /projects/{projectName}/mirrors/{mirrorId}/run + * + *

Runs the mirroring task immediately. + */ + @RequiresWritePermission(repository = Project.REPO_META) + // Mirroring may be a long-running task, so we need to increase the timeout. + @RequestTimeout(value = 5, unit = TimeUnit.MINUTES) + @Post("/projects/{projectName}/mirrors/{mirrorId}/run") + public CompletableFuture runMirror(@Param String projectName, @Param String mirrorId) + throws Exception { + return mirrorRunner.run(projectName, mirrorId); + } + private static MirrorDto convertToMirrorDto(String projectName, Mirror mirror) { final URI remoteRepoUri = mirror.remoteRepoUri(); + final Cron schedule = mirror.schedule(); + final String scheduleStr = schedule != null ? schedule.asString() : null; return new MirrorDto(mirror.id(), mirror.enabled(), projectName, - mirror.schedule().asString(), + scheduleStr, mirror.direction().name(), mirror.localRepo().name(), mirror.localPath(), diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java index 9aa981ec40..04dccd857a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java @@ -39,6 +39,7 @@ import com.linecorp.centraldogma.internal.CsrfToken; import com.linecorp.centraldogma.server.internal.admin.auth.AuthUtil; import com.linecorp.centraldogma.server.internal.admin.service.TokenNotFoundException; +import com.linecorp.centraldogma.server.internal.api.HttpApiUtil; import com.linecorp.centraldogma.server.metadata.Token; import com.linecorp.centraldogma.server.metadata.Tokens; import com.linecorp.centraldogma.server.metadata.User; @@ -63,6 +64,7 @@ public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest final OAuth2Token token = AuthTokenExtractors.oAuth2().apply(data.headers()); if (token != null && token.accessToken().equals(CsrfToken.ANONYMOUS)) { AuthUtil.setCurrentUser(ctx, User.ANONYMOUS); + HttpApiUtil.setVerboseResponses(ctx, User.ANONYMOUS); return completedFuture(true); } @@ -81,8 +83,9 @@ public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest login.append('@').append(((InetSocketAddress) ra).getHostString()); } ctx.logBuilder().authenticatedUser("app/" + appId); - AuthUtil.setCurrentUser( - ctx, new UserWithToken(login.toString(), appToken)); + final UserWithToken user = new UserWithToken(login.toString(), appToken); + AuthUtil.setCurrentUser(ctx, user); + HttpApiUtil.setVerboseResponses(ctx, user); res.complete(true); } else { res.complete(false); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java index 155a8dbaa6..e711c9fdb6 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java @@ -21,6 +21,7 @@ import java.io.File; import java.net.URI; +import java.time.Instant; import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.Objects; @@ -36,11 +37,13 @@ import com.google.common.base.MoreObjects.ToStringHelper; import com.linecorp.centraldogma.common.Author; -import com.linecorp.centraldogma.server.MirrorException; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.credential.Credential; import com.linecorp.centraldogma.server.mirror.Mirror; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirrorStatus; import com.linecorp.centraldogma.server.storage.repository.Repository; public abstract class AbstractMirror implements Mirror { @@ -51,7 +54,6 @@ public abstract class AbstractMirror implements Mirror { private final String id; private final boolean enabled; - private final Cron schedule; private final MirrorDirection direction; private final Credential credential; private final Repository localRepo; @@ -61,16 +63,18 @@ public abstract class AbstractMirror implements Mirror { private final String remoteBranch; @Nullable private final String gitignore; + @Nullable + private final Cron schedule; + @Nullable private final ExecutionTime executionTime; private final long jitterMillis; - protected AbstractMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + protected AbstractMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { this.id = requireNonNull(id, "id"); this.enabled = enabled; - this.schedule = requireNonNull(schedule, "schedule"); this.direction = requireNonNull(direction, "direction"); this.credential = requireNonNull(credential, "credential"); this.localRepo = requireNonNull(localRepo, "localRepo"); @@ -80,14 +84,21 @@ protected AbstractMirror(String id, boolean enabled, Cron schedule, MirrorDirect this.remoteBranch = requireNonNull(remoteBranch, "remoteBranch"); this.gitignore = gitignore; - executionTime = ExecutionTime.forCron(this.schedule); - - // Pre-calculate a constant jitter value up to 1 minute for a mirror. - // Use the properties' hash code so that the same properties result in the same jitter. - jitterMillis = Math.abs(Objects.hash(this.schedule.asString(), this.direction, - this.localRepo.parent().name(), this.localRepo.name(), - this.remoteRepoUri, this.remotePath, this.remoteBranch) / - (Integer.MAX_VALUE / 60000)); + if (schedule != null) { + this.schedule = requireNonNull(schedule, "schedule"); + executionTime = ExecutionTime.forCron(this.schedule); + + // Pre-calculate a constant jitter value up to 1 minute for a mirror. + // Use the properties' hash code so that the same properties result in the same jitter. + jitterMillis = Math.abs(Objects.hash(this.schedule.asString(), this.direction, + this.localRepo.parent().name(), this.localRepo.name(), + this.remoteRepoUri, this.remotePath, this.remoteBranch) / + (Integer.MAX_VALUE / 60000)); + } else { + this.schedule = null; + executionTime = null; + jitterMillis = -1; + } } @Override @@ -164,37 +175,51 @@ public final boolean enabled() { } @Override - public final void mirror(File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes) { + public final MirrorResult mirror(File workDir, CommandExecutor executor, int maxNumFiles, + long maxNumBytes) { + final Instant triggeredTime = Instant.now(); try { switch (direction()) { case LOCAL_TO_REMOTE: - mirrorLocalToRemote(workDir, maxNumFiles, maxNumBytes); - break; + return mirrorLocalToRemote(workDir, maxNumFiles, maxNumBytes, triggeredTime); case REMOTE_TO_LOCAL: - mirrorRemoteToLocal(workDir, executor, maxNumFiles, maxNumBytes); - break; + return mirrorRemoteToLocal(workDir, executor, maxNumFiles, maxNumBytes, triggeredTime); + default: + throw new Error("Should never reach here"); } } catch (InterruptedException e) { // Propagate the interruption. Thread.currentThread().interrupt(); + throw new MirrorException(e); } catch (MirrorException e) { throw e; } catch (Exception e) { - throw new MirrorException(e); + final String message = e.getMessage(); + if (message != null) { + throw new MirrorException(message, e); + } else { + throw new MirrorException(e); + } } } - protected abstract void mirrorLocalToRemote( - File workDir, int maxNumFiles, long maxNumBytes) throws Exception; + protected abstract MirrorResult mirrorLocalToRemote( + File workDir, int maxNumFiles, long maxNumBytes, Instant triggeredTime) throws Exception; + + protected abstract MirrorResult mirrorRemoteToLocal( + File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws Exception; - protected abstract void mirrorRemoteToLocal( - File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes) throws Exception; + protected final MirrorResult newMirrorResult(MirrorStatus mirrorStatus, @Nullable String description, + Instant triggeredTime) { + return new MirrorResult(id, localRepo.parent().name(), localRepo.name(), mirrorStatus, description, + triggeredTime); + } @Override public String toString() { final ToStringHelper helper = MoreObjects.toStringHelper("") .omitNullValues() - .add("schedule", CronDescriptor.instance().describe(schedule)) .add("direction", direction) .add("localProj", localRepo.parent().name()) .add("localRepo", localRepo.name()) @@ -204,7 +229,9 @@ public String toString() { .add("remoteBranch", remoteBranch) .add("gitignore", gitignore) .add("credential", credential); - + if (schedule != null) { + helper.add("schedule", CronDescriptor.instance().describe(schedule)); + } return helper.toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java index f0a84c7a5e..73392970f3 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java @@ -20,6 +20,7 @@ import java.io.File; import java.net.URI; +import java.time.Instant; import javax.annotation.Nullable; @@ -28,6 +29,7 @@ import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.credential.Credential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; import com.linecorp.centraldogma.server.storage.repository.Repository; public final class CentralDogmaMirror extends AbstractMirror { @@ -56,13 +58,16 @@ String remoteRepo() { } @Override - protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes, + Instant triggeredTime) + throws Exception { throw new UnsupportedOperationException(); } @Override - protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, - int maxNumFiles, long maxNumBytes) throws Exception { + protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executor, + int maxNumFiles, long maxNumBytes, Instant triggeredTime) + throws Exception { throw new UnsupportedOperationException(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java index 168f81301c..fc71360940 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java @@ -34,7 +34,7 @@ public final class DefaultMirroringServicePlugin implements Plugin { @Nullable - private volatile DefaultMirroringService mirroringService; + private volatile MirrorSchedulingService mirroringService; @Override public PluginTarget target() { @@ -45,7 +45,7 @@ public PluginTarget target() { public synchronized CompletionStage start(PluginContext context) { requireNonNull(context, "context"); - DefaultMirroringService mirroringService = this.mirroringService; + MirrorSchedulingService mirroringService = this.mirroringService; if (mirroringService == null) { final CentralDogmaConfig cfg = context.config(); final MirroringServicePluginConfig mirroringServicePluginConfig = @@ -63,7 +63,7 @@ public synchronized CompletionStage start(PluginContext context) { maxNumFilesPerMirror = MirroringServicePluginConfig.INSTANCE.maxNumFilesPerMirror(); maxNumBytesPerMirror = MirroringServicePluginConfig.INSTANCE.maxNumBytesPerMirror(); } - mirroringService = new DefaultMirroringService(new File(cfg.dataDir(), "_mirrors"), + mirroringService = new MirrorSchedulingService(new File(cfg.dataDir(), "_mirrors"), context.projectManager(), context.meterRegistry(), numThreads, @@ -77,7 +77,7 @@ public synchronized CompletionStage start(PluginContext context) { @Override public synchronized CompletionStage stop(PluginContext context) { - final DefaultMirroringService mirroringService = this.mirroringService; + final MirrorSchedulingService mirroringService = this.mirroringService; if (mirroringService != null && mirroringService.isStarted()) { mirroringService.stop(); } @@ -90,7 +90,7 @@ public Class configType() { } @Nullable - public DefaultMirroringService mirroringService() { + public MirrorSchedulingService mirroringService() { return mirroringService; } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java new file mode 100644 index 0000000000..f9d4eec804 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java @@ -0,0 +1,154 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.mirror; + +import static com.linecorp.centraldogma.server.internal.ExecutorServiceUtil.terminate; + +import java.io.File; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.common.util.SafeCloseable; +import com.linecorp.centraldogma.common.MirrorException; +import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirroringServicePluginConfig; +import com.linecorp.centraldogma.server.storage.repository.MetaRepository; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics; +import io.netty.util.concurrent.DefaultThreadFactory; + +public final class MirrorRunner implements SafeCloseable { + + private final ProjectApiManager projectApiManager; + private final CommandExecutor commandExecutor; + private final File workDir; + private final MirroringServicePluginConfig mirrorConfig; + private final ExecutorService worker; + + private final Map> inflightRequests = new ConcurrentHashMap<>(); + + public MirrorRunner(ProjectApiManager projectApiManager, CommandExecutor commandExecutor, + CentralDogmaConfig cfg, MeterRegistry meterRegistry) { + this.projectApiManager = projectApiManager; + this.commandExecutor = commandExecutor; + // TODO(ikhoon): Periodically clean up stale repositories. + workDir = new File(cfg.dataDir(), "_mirrors_manual"); + MirroringServicePluginConfig mirrorConfig = + (MirroringServicePluginConfig) cfg.pluginConfigMap() + .get(MirroringServicePluginConfig.class); + if (mirrorConfig == null) { + mirrorConfig = MirroringServicePluginConfig.INSTANCE; + } + this.mirrorConfig = mirrorConfig; + + final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor( + 0, mirrorConfig.numMirroringThreads(), + // TODO(minwoox): Use LinkedTransferQueue when we upgrade to JDK 21. + 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), + new DefaultThreadFactory("mirror-api-worker", true)); + threadPoolExecutor.allowCoreThreadTimeOut(true); + worker = ExecutorServiceMetrics.monitor(meterRegistry, threadPoolExecutor, + "mirrorApiWorker"); + } + + public CompletableFuture run(String projectName, String mirrorId) { + // If there is an inflight request, return it to avoid running the same mirror task multiple times. + return inflightRequests.computeIfAbsent(new MirrorKey(projectName, mirrorId), this::run); + } + + private CompletableFuture run(MirrorKey mirrorKey) { + try { + final CompletableFuture future = + metaRepo(mirrorKey.projectName).mirror(mirrorKey.mirrorId).thenApplyAsync(mirror -> { + if (!mirror.enabled()) { + throw new MirrorException("The mirror is disabled: " + mirrorKey); + } + + return mirror.mirror(workDir, commandExecutor, + mirrorConfig.maxNumFilesPerMirror(), + mirrorConfig.maxNumBytesPerMirror()); + }, worker); + // Remove the inflight request when the mirror task is done. + future.handleAsync((unused0, unused1) -> inflightRequests.remove(mirrorKey), worker); + return future; + } catch (Throwable e) { + inflightRequests.remove(mirrorKey); + throw e; + } + } + + private MetaRepository metaRepo(String projectName) { + return projectApiManager.getProject(projectName).metaRepo(); + } + + @Override + public void close() { + final boolean interrupted = terminate(worker); + if (interrupted) { + Thread.currentThread().interrupt(); + } + inflightRequests.clear(); + } + + private static final class MirrorKey { + private final String projectName; + private final String mirrorId; + + private MirrorKey(String projectName, String mirrorId) { + this.projectName = projectName; + this.mirrorId = mirrorId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MirrorKey)) { + return false; + } + final MirrorKey mirrorKey = (MirrorKey) o; + return projectName.equals(mirrorKey.projectName) && + mirrorId.equals(mirrorKey.mirrorId); + } + + @Override + public int hashCode() { + return Objects.hash(projectName, mirrorId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("projectName", projectName) + .add("mirrorId", mirrorId) + .toString(); + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java similarity index 95% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java index 6076ed77a1..ea5cd3c476 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java @@ -47,7 +47,7 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.linecorp.centraldogma.server.MirrorException; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.server.MirroringService; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.mirror.Mirror; @@ -58,9 +58,9 @@ import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics; import io.netty.util.concurrent.DefaultThreadFactory; -public final class DefaultMirroringService implements MirroringService { +public final class MirrorSchedulingService implements MirroringService { - private static final Logger logger = LoggerFactory.getLogger(DefaultMirroringService.class); + private static final Logger logger = LoggerFactory.getLogger(MirrorSchedulingService.class); /** * How often to check the mirroring schedules. i.e. every second. @@ -80,7 +80,7 @@ public final class DefaultMirroringService implements MirroringService { private ZonedDateTime lastExecutionTime; private final MeterRegistry meterRegistry; - DefaultMirroringService(File workDir, ProjectManager projectManager, MeterRegistry meterRegistry, + MirrorSchedulingService(File workDir, ProjectManager projectManager, MeterRegistry meterRegistry, int numThreads, int maxNumFilesPerMirror, long maxNumBytesPerMirror) { this.workDir = requireNonNull(workDir, "workDir"); @@ -185,7 +185,10 @@ private void schedulePendingMirrors() { logger.warn("Failed to load the mirror list from: {}", project.name(), e); return; } - mirrors.forEach(m -> { + for (Mirror m : mirrors) { + if (m.schedule() == null) { + continue; + } try { if (m.nextExecutionTime(currentLastExecutionTime).compareTo(now) < 0) { run(project, m); @@ -193,7 +196,7 @@ private void schedulePendingMirrors() { } catch (Exception e) { logger.warn("Unexpected exception while mirroring: {}", m, e); } - }); + } }); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java index 58db07d632..dfc0c5e5ed 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java @@ -292,10 +292,13 @@ private Command newCommand(Credential credential, Author author, S private static void validateMirror(MirrorDto mirror) { checkArgument(!Strings.isNullOrEmpty(mirror.id()), "Mirror ID is empty"); - final Cron schedule = MirrorConfig.CRON_PARSER.parse(mirror.schedule()); - final CronField secondField = schedule.retrieve(CronFieldName.SECOND); - checkArgument(!secondField.getExpression().asString().contains("*"), - "The second field of the schedule must be specified. (seconds: *, expected: 0-59)"); + final String scheduleString = mirror.schedule(); + if (scheduleString != null) { + final Cron schedule = MirrorConfig.CRON_PARSER.parse(scheduleString); + final CronField secondField = schedule.retrieve(CronFieldName.SECOND); + checkArgument(!secondField.getExpression().asString().contains("*"), + "The second field of the schedule must be specified. (seconds: *, expected: 0-59)"); + } } private static MirrorConfig converterToMirrorConfig(MirrorDto mirrorDto) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java index a553dac741..58a7ef94e9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java @@ -78,8 +78,8 @@ public final class MirrorConfig { private final URI remoteUri; @Nullable private final String gitignore; - @Nullable private final String credentialId; + @Nullable private final Cron schedule; @JsonCreator @@ -91,10 +91,14 @@ public MirrorConfig(@JsonProperty("id") String id, @JsonProperty("localPath") @Nullable String localPath, @JsonProperty(value = "remoteUri", required = true) URI remoteUri, @JsonProperty("gitignore") @Nullable Object gitignore, - @JsonProperty("credentialId") @Nullable String credentialId) { + @JsonProperty("credentialId") String credentialId) { this.id = requireNonNull(id, "id"); this.enabled = firstNonNull(enabled, true); - this.schedule = CRON_PARSER.parse(firstNonNull(schedule, DEFAULT_SCHEDULE)); + if (schedule != null) { + this.schedule = CRON_PARSER.parse(schedule); + } else { + this.schedule = null; + } this.direction = requireNonNull(direction, "direction"); this.localRepo = requireNonNull(localRepo, "localRepo"); this.localPath = firstNonNull(localPath, "/"); @@ -117,7 +121,7 @@ public MirrorConfig(@JsonProperty("id") String id, } else { this.gitignore = null; } - this.credentialId = credentialId; + this.credentialId = requireNonNull(credentialId, "credentialId"); } @Nullable @@ -195,9 +199,14 @@ public String credentialId() { return credentialId; } + @Nullable @JsonProperty("schedule") public String schedule() { - return schedule.asString(); + if (schedule != null) { + return schedule.asString(); + } else { + return null; + } } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java index 4732a57b3f..68f930221f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java @@ -23,7 +23,7 @@ import com.cronutils.model.Cron; -import com.linecorp.centraldogma.server.MirrorException; +import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.credential.Credential; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -40,7 +40,9 @@ public interface Mirror { /** * Returns the schedule for the mirroring task. + * {@code null} if the mirroring task is not scheduled. */ + @Nullable Cron schedule(); /** @@ -108,5 +110,5 @@ public interface Mirror { * @param maxNumBytes the maximum bytes allowed to the mirroring task. A {@link MirrorException} would be * raised if the total size of the files to be mirrored exceeds it. */ - void mirror(File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes); + MirrorResult mirror(File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java index a5ea678dfe..56dae01833 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java @@ -35,6 +35,7 @@ public final class MirrorContext { private final String id; private final boolean enabled; + @Nullable private final Cron schedule; private final MirrorDirection direction; private final Credential credential; @@ -47,12 +48,12 @@ public final class MirrorContext { /** * Creates a new instance. */ - public MirrorContext(String id, boolean enabled, Cron schedule, MirrorDirection direction, + public MirrorContext(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteUri, @Nullable String gitignore) { this.id = requireNonNull(id, "id"); this.enabled = enabled; - this.schedule = requireNonNull(schedule, "schedule"); + this.schedule = schedule; this.direction = requireNonNull(direction, "direction"); this.credential = requireNonNull(credential, "credential"); this.localRepo = requireNonNull(localRepo, "localRepo"); @@ -77,7 +78,9 @@ public boolean enabled() { /** * Returns the cron schedule of this mirror. + * {@code null} if this mirror is not scheduled automatically. */ + @Nullable public Cron schedule() { return schedule; } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java new file mode 100644 index 0000000000..e3c9ab1acf --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java @@ -0,0 +1,146 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.mirror; + +import static java.util.Objects.requireNonNull; + +import java.time.Instant; +import java.util.Objects; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +/** + * The result of a mirroring operation. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public final class MirrorResult { + + private final String mirrorId; + private final String projectName; + private final String repoName; + private final MirrorStatus mirrorStatus; + @Nullable + private final String description; + private final Instant triggeredTime; + + /** + * Creates a new instance. + */ + @JsonCreator + public MirrorResult(@JsonProperty("mirrorId") String mirrorId, + @JsonProperty("projectName") String projectName, + @JsonProperty("repoName") String repoName, + @JsonProperty("mirrorStatus") MirrorStatus mirrorStatus, + @JsonProperty("description") @Nullable String description, + @JsonProperty("triggeredTime") Instant triggeredTime) { + this.mirrorId = requireNonNull(mirrorId, "mirrorId"); + this.projectName = requireNonNull(projectName, "projectName"); + this.repoName = requireNonNull(repoName, "repoName"); + this.mirrorStatus = requireNonNull(mirrorStatus, "mirrorStatus"); + this.description = description; + this.triggeredTime = requireNonNull(triggeredTime, "triggeredTime"); + } + + /** + * Returns the ID of the mirror. + */ + @JsonProperty("mirrorId") + public String mirrorId() { + return mirrorId; + } + + /** + * Returns the project name which {@link #mirrorId()} belongs to. + */ + @JsonProperty("projectName") + public String projectName() { + return projectName; + } + + /** + * Returns the repository name where the mirroring operation is performed. + */ + @JsonProperty("repoName") + public String repoName() { + return repoName; + } + + /** + * Returns the status of the mirroring operation. + */ + @JsonProperty("mirrorStatus") + public MirrorStatus mirrorStatus() { + return mirrorStatus; + } + + /** + * Returns the description of the mirroring operation. + */ + @Nullable + @JsonProperty("description") + public String description() { + return description; + } + + /** + * Returns the time when the mirroring operation was triggered. + */ + @JsonProperty("triggeredTime") + public Instant triggeredTime() { + return triggeredTime; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MirrorResult)) { + return false; + } + final MirrorResult that = (MirrorResult) o; + return mirrorId.equals(that.mirrorId) && + projectName.equals(that.projectName) && + repoName.equals(that.repoName) && + mirrorStatus == that.mirrorStatus && + Objects.equals(description, that.description) && + triggeredTime.equals(triggeredTime); + } + + @Override + public int hashCode() { + return Objects.hash(mirrorId, projectName, repoName, mirrorStatus, description, triggeredTime); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .omitNullValues() + .add("mirrorId", mirrorId) + .add("projectName", projectName) + .add("repoName", repoName) + .add("mirrorStatus", mirrorStatus) + .add("description", description) + .add("triggeredTime", triggeredTime) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorStatus.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorStatus.java new file mode 100644 index 0000000000..bbeafff6ac --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorStatus.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.mirror; + +/** + * The status of a mirroring operation. + */ +public enum MirrorStatus { + /** + * The mirroring was successful. + */ + SUCCESS, + /** + * The target repository was already up-to-date. + */ + UP_TO_DATE, +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1Test.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1Test.java index 47e2479ce8..07c58900e0 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1Test.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1Test.java @@ -129,7 +129,8 @@ void emptyChangeSet() { assertThatJson(res.contentUtf8()).isEqualTo( '{' + " \"exception\": \"" + RedundantChangeException.class.getName() + "\"," + - " \"message\": \"${json-unit.ignore}\"" + + " \"message\": \"${json-unit.ignore}\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'); } @@ -634,7 +635,8 @@ void deleteFileInvalidRevision() { assertThatJson(res.contentUtf8()).isEqualTo( '{' + " \"exception\": \"" + ChangeConflictException.class.getName() + "\"," + - " \"message\": \"${json-unit.ignore}\"" + + " \"message\": \"${json-unit.ignore}\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'); } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/MergeFileTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/MergeFileTest.java index a56a38cdf7..d92d2e598a 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/MergeFileTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/MergeFileTest.java @@ -85,7 +85,8 @@ void mergeJsonFiles() { expectedJson = '{' + " \"exception\": \"com.linecorp.centraldogma.common.EntryNotFoundException\"," + - " \"message\": \"Entry '/foo3.json (revision: 4)' does not exist.\"" + + " \"message\": \"Entry '/foo3.json (revision: 4)' does not exist.\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'; assertThatJson(aRes.contentUtf8()).isEqualTo(expectedJson); } @@ -103,7 +104,8 @@ void exceptionWhenOnlyOptionalFilesAndDoNotExist() { final String expectedJson = '{' + " \"exception\": \"com.linecorp.centraldogma.common.EntryNotFoundException\"," + - " \"message\": \"Entry '/no_exist1.json,/no_exist2.json (revision: 4)' does not exist.\"" + + " \"message\": \"Entry '/no_exist1.json,/no_exist2.json (revision: 4)' does not exist.\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'; assertThatJson(aRes.contentUtf8()).isEqualTo(expectedJson); } @@ -137,7 +139,8 @@ void mergeJsonPaths() { expectedJson = '{' + " \"exception\": \"com.linecorp.centraldogma.common.QueryExecutionException\"," + - " \"message\": \"JSON path evaluation failed: $.c\"" + + " \"message\": \"JSON path evaluation failed: $.c\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'; assertThatJson(aRes.contentUtf8()).isEqualTo(expectedJson); } @@ -191,7 +194,8 @@ void mismatchedValueWhileMerging() { final String expectedJson = '{' + " \"exception\": \"com.linecorp.centraldogma.common.QueryExecutionException\"," + - " \"message\": \"Failed to merge tree. /a/ type: NUMBER (expected: STRING)\"" + + " \"message\": \"Failed to merge tree. /a/ type: NUMBER (expected: STRING)\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'; assertThatJson(aRes.contentUtf8()).isEqualTo(expectedJson); } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java index ca9660dbec..7a7e8531b5 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java @@ -98,7 +98,8 @@ void createRepositoryWithSameName() { final String expectedJson = '{' + " \"exception\": \"" + RepositoryExistsException.class.getName() + "\"," + - " \"message\": \"Repository 'myPro/myNewRepo' exists already.\"" + + " \"message\": \"Repository 'myPro/myNewRepo' exists already.\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'; assertThatJson(aRes.contentUtf8()).isEqualTo(expectedJson); } @@ -122,7 +123,8 @@ void createRepositoryInAbsentProject() { final String expectedJson = '{' + " \"exception\": \"" + ProjectNotFoundException.class.getName() + "\"," + - " \"message\": \"Project 'absentProject' does not exist.\"" + + " \"message\": \"Project 'absentProject' does not exist.\"," + + " \"detail\": \"${json-unit.ignore}\"" + '}'; assertThatJson(aRes.contentUtf8()).isEqualTo(expectedJson); } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java index db02703f7b..d6b9e9cf35 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java @@ -207,7 +207,7 @@ void test_anonymous() { final WebClient client2 = WebClient.builder(server.httpUri()).build(); final AggregatedHttpResponse response2 = client2.get("/projects/project1/repos/anonymous_allowed_repo") - .aggregate().join(); + .aggregate().join(); assertThat(response2.status()).isEqualTo(HttpStatus.UNAUTHORIZED); } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePluginTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServicePluginTest.java similarity index 98% rename from server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePluginTest.java rename to server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServicePluginTest.java index d2c717cd1f..59ea58c38a 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePluginTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServicePluginTest.java @@ -22,7 +22,7 @@ import com.linecorp.centraldogma.server.CentralDogmaConfig; import com.linecorp.centraldogma.server.mirror.MirroringServicePluginConfig; -class DefaultMirroringServicePluginTest { +class MirrorSchedulingServicePluginTest { @Test void pluginConfig() throws Exception { diff --git a/webapp/package-lock.json b/webapp/package-lock.json index f198216af4..9d28d4466a 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -16,6 +16,7 @@ "@tanstack/react-table": "^8.19.2", "axios": "1.7.2", "chakra-react-select": "^4.9.1", + "cronstrue": "^2.50.0", "date-fns": "^3.6.0", "framer-motion": "^11.2.13", "jsonpath": "^1.1.1", @@ -6137,6 +6138,15 @@ "dev": true, "license": "MIT" }, + "node_modules/cronstrue": { + "version": "2.50.0", + "resolved": "https://registry.npmjs.org/cronstrue/-/cronstrue-2.50.0.tgz", + "integrity": "sha512-ULYhWIonJzlScCCQrPUG5uMXzXxSixty4djud9SS37DoNxDdkeRocxzHuAo4ImRBUK+mAuU5X9TSwEDccnnuPg==", + "license": "MIT", + "bin": { + "cronstrue": "bin/cli.js" + } + }, "node_modules/cross-env": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", diff --git a/webapp/package.json b/webapp/package.json index decb1a2206..f5d02cbcac 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -29,6 +29,7 @@ "@tanstack/react-table": "^8.19.2", "axios": "1.7.2", "chakra-react-select": "^4.9.1", + "cronstrue": "^2.50.0", "date-fns": "^3.6.0", "framer-motion": "^11.2.13", "jsonpath": "^1.1.1", diff --git a/webapp/src/dogma/common/components/Navbar.tsx b/webapp/src/dogma/common/components/Navbar.tsx index 7d81ce20a6..ebe2a85f20 100644 --- a/webapp/src/dogma/common/components/Navbar.tsx +++ b/webapp/src/dogma/common/components/Navbar.tsx @@ -206,8 +206,8 @@ export const Navbar = () => { if (typeof window !== 'undefined') { Router.push( process.env.NEXT_PUBLIC_HOST - ? `${process.env.NEXT_PUBLIC_HOST}/link/auth/login/?return_to=${window.location.origin}` - : `/link/auth/login/`, + ? `${process.env.NEXT_PUBLIC_HOST}/link/auth/login?return_to=${window.location.origin}` + : `/link/auth/login`, ); } }} diff --git a/webapp/src/dogma/common/components/NotificationWrapper.tsx b/webapp/src/dogma/common/components/NotificationWrapper.tsx index 2f0ffb0ba6..f8c048ff30 100644 --- a/webapp/src/dogma/common/components/NotificationWrapper.tsx +++ b/webapp/src/dogma/common/components/NotificationWrapper.tsx @@ -3,18 +3,19 @@ import { useToast } from '@chakra-ui/react'; import { useAppSelector } from 'dogma/hooks'; export const NotificationWrapper = (props: { children: ReactNode }) => { - const { title, text, type, timestamp } = useAppSelector((state) => state.notification); + const { title, description, type, containerStyle, timestamp } = useAppSelector((state) => state.notification); const toast = useToast(); useEffect(() => { - if (text) { + if (timestamp) { toast({ - title: title, - description: text, + title, + description, status: type, duration: 10000, isClosable: true, + containerStyle, }); } - }, [title, text, type, timestamp, toast]); + }, [title, description, type, timestamp, containerStyle, toast]); return <>{props.children}; }; diff --git a/webapp/src/dogma/features/api/apiSlice.ts b/webapp/src/dogma/features/api/apiSlice.ts index 6e576c91d7..49d3a3b56d 100644 --- a/webapp/src/dogma/features/api/apiSlice.ts +++ b/webapp/src/dogma/features/api/apiSlice.ts @@ -30,6 +30,7 @@ import { AddUserPermissionDto } from 'dogma/features/repo/permissions/AddUserPer import { DeleteMemberDto } from 'dogma/features/project/settings/members/DeleteMemberDto'; import { MirrorDto } from 'dogma/features/project/settings/mirrors/MirrorDto'; import { CredentialDto } from 'dogma/features/project/settings/credentials/CredentialDto'; +import { MirrorResult } from '../mirror/MirrorResult'; export type ApiAction = { (arg: Arg): { unwrap: () => Promise }; @@ -346,6 +347,13 @@ export const apiSlice = createApi({ }), invalidatesTags: ['Metadata'], }), + runMirror: builder.mutation({ + query: ({ projectName, id }) => ({ + url: `/api/v1/projects/${projectName}/mirrors/${id}/run`, + method: 'POST', + }), + invalidatesTags: ['Metadata'], + }), getCredentials: builder.query({ query: (projectName) => `/api/v1/projects/${projectName}/credentials`, providesTags: ['Metadata'], @@ -422,6 +430,7 @@ export const { useGetMirrorQuery, useAddNewMirrorMutation, useUpdateMirrorMutation, + useRunMirrorMutation, // Credential useGetCredentialsQuery, useGetCredentialQuery, diff --git a/webapp/src/dogma/features/auth/Authorized.tsx b/webapp/src/dogma/features/auth/Authorized.tsx index 58283474ed..355955f177 100644 --- a/webapp/src/dogma/features/auth/Authorized.tsx +++ b/webapp/src/dogma/features/auth/Authorized.tsx @@ -47,7 +47,7 @@ export const Authorized = (props: { children: ReactNode }) => { if (typeof window !== 'undefined') { router.push( process.env.NEXT_PUBLIC_HOST - ? `${process.env.NEXT_PUBLIC_HOST}/link/auth/login/?return_to=${window.location.origin}` + ? `${process.env.NEXT_PUBLIC_HOST}/link/auth/login?return_to=${window.location.origin}` : `/link/auth/login`, ); } diff --git a/webapp/src/dogma/features/mirror/MirrorResult.ts b/webapp/src/dogma/features/mirror/MirrorResult.ts new file mode 100644 index 0000000000..6ae480c43b --- /dev/null +++ b/webapp/src/dogma/features/mirror/MirrorResult.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +export type MirrorStatus = 'SUCCESS' | 'UP_TO_DATE'; + +export interface MirrorResult { + mirrorId: string; + projectName: string; + repoName: string; + mirrorStatus: MirrorStatus; + description: string; +} diff --git a/webapp/src/dogma/features/mirror/RunMirrorButton.tsx b/webapp/src/dogma/features/mirror/RunMirrorButton.tsx new file mode 100644 index 0000000000..c35eaabe12 --- /dev/null +++ b/webapp/src/dogma/features/mirror/RunMirrorButton.tsx @@ -0,0 +1,115 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import { useRunMirrorMutation } from 'dogma/features/api/apiSlice'; +import { newNotification } from 'dogma/features/notification/notificationSlice'; +import { useAppDispatch } from 'dogma/hooks'; +import { SerializedError } from '@reduxjs/toolkit'; +import { FetchBaseQueryError } from '@reduxjs/toolkit/query'; +import ErrorMessageParser from 'dogma/features/services/ErrorMessageParser'; +import { MirrorResult } from './MirrorResult'; +import { MirrorDto } from '../project/settings/mirrors/MirrorDto'; +import { + Button, + ButtonGroup, + Mark, + Popover, + PopoverArrow, + PopoverBody, + PopoverCloseButton, + PopoverContent, + PopoverFooter, + PopoverHeader, + PopoverTrigger, + Text, + useDisclosure, +} from '@chakra-ui/react'; +import { ReactNode } from 'react'; + +type RunMirrorProps = { + mirror: MirrorDto; + children: ({ isLoading }: { isLoading: boolean; onToggle: () => void }) => ReactNode; +}; +export const RunMirror = ({ mirror, children }: RunMirrorProps) => { + const [runMirror, { isLoading }] = useRunMirrorMutation(); + const { isOpen, onOpen, onClose, onToggle } = useDisclosure(); + const dispatch = useAppDispatch(); + const onClick = async () => { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const response: any = await runMirror({ projectName: mirror.projectName, id: mirror.id }).unwrap(); + if ((response as { error: FetchBaseQueryError | SerializedError }).error) { + throw (response as { error: FetchBaseQueryError | SerializedError }).error; + } + const result: MirrorResult = response; + if (result.mirrorStatus === 'SUCCESS') { + dispatch( + newNotification(`Mirror ${mirror.id} is performed successfully`, result.description, 'success'), + ); + } else if (result.mirrorStatus === 'UP_TO_DATE') { + dispatch(newNotification(`No changes`, result.description, 'info')); + } + onClose(); + } catch (error) { + if (error.data && error.data.exception) { + // A exception is thrown by the backend + let detail = null; + let containerStyle = null; + if (error.data.detail) { + detail = ${error.data.detail.replaceAll('\t', ' '.repeat(4))}; + containerStyle = { + maxWidth: '50%', + }; + } + dispatch(newNotification(error.data.message, detail, 'error', containerStyle)); + } else { + dispatch( + newNotification(`Failed to run mirror ${mirror.id}`, ErrorMessageParser.parse(error), 'error'), + ); + } + } + }; + + return ( + + {children({ isLoading, onToggle })} + + + Confirmation + + + + + Do you want to run{' '} + + {mirror.id} + {' '} + mirror? + + + + + + + + + + ); +}; diff --git a/webapp/src/dogma/features/notification/notificationSlice.ts b/webapp/src/dogma/features/notification/notificationSlice.ts index 8c6ded8769..d0406b9454 100644 --- a/webapp/src/dogma/features/notification/notificationSlice.ts +++ b/webapp/src/dogma/features/notification/notificationSlice.ts @@ -1,19 +1,23 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; +import { ReactNode } from 'react'; +import { StyleProps } from '@chakra-ui/system'; type NotificationType = 'error' | 'info' | 'warning' | 'success' | 'loading'; export interface Notification { title: string; - text: string; + description: string | ReactNode; type: NotificationType; + containerStyle?: StyleProps; timestamp: number; } const initialState: Notification = { title: '', - text: '', + description: '', type: 'info', - timestamp: Date.now(), + containerStyle: {}, + timestamp: null, }; export const notificationSlice = createSlice({ @@ -22,8 +26,9 @@ export const notificationSlice = createSlice({ reducers: { createNotification(state: Notification, action: PayloadAction) { state.title = action.payload.title; - state.text = action.payload.text; + state.description = action.payload.description; state.type = action.payload.type; + state.containerStyle = action.payload.containerStyle; state.timestamp = action.payload.timestamp; }, resetState(state: Notification) { @@ -35,6 +40,11 @@ export const notificationSlice = createSlice({ export const { createNotification, resetState } = notificationSlice.actions; export const notificationReducer = notificationSlice.reducer; -export function newNotification(title: string, text: string, type: NotificationType) { - return createNotification({ title, text, type, timestamp: Date.now() }); +export function newNotification( + title: string, + description: string | ReactNode, + type: NotificationType, + containerStyle?: StyleProps, +): PayloadAction { + return createNotification({ title, description, type, containerStyle, timestamp: Date.now() }); } diff --git a/webapp/src/dogma/features/project/settings/mirrors/MirrorDto.ts b/webapp/src/dogma/features/project/settings/mirrors/MirrorDto.ts index b4fec4a967..cea47b7040 100644 --- a/webapp/src/dogma/features/project/settings/mirrors/MirrorDto.ts +++ b/webapp/src/dogma/features/project/settings/mirrors/MirrorDto.ts @@ -1,7 +1,7 @@ export interface MirrorDto { id: string; projectName: string; - schedule: string; + schedule?: string; direction: 'REMOTE_TO_LOCAL' | 'LOCAL_TO_REMOTE'; localRepo: string; localPath: string; diff --git a/webapp/src/dogma/features/project/settings/mirrors/MirrorForm.tsx b/webapp/src/dogma/features/project/settings/mirrors/MirrorForm.tsx index de4e2b5881..e7e86caa17 100644 --- a/webapp/src/dogma/features/project/settings/mirrors/MirrorForm.tsx +++ b/webapp/src/dogma/features/project/settings/mirrors/MirrorForm.tsx @@ -16,6 +16,8 @@ import { Controller, useForm, UseFormSetError } from 'react-hook-form'; import { + Alert, + AlertIcon, Button, Center, Divider, @@ -43,21 +45,18 @@ import { GoArrowBoth, GoArrowDown, GoArrowUp, GoKey, GoRepo } from 'react-icons/ import { Select } from 'chakra-react-select'; import { IoBanSharp } from 'react-icons/io5'; import { useGetCredentialsQuery, useGetReposQuery } from 'dogma/features/api/apiSlice'; -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import FieldErrorMessage from 'dogma/common/components/form/FieldErrorMessage'; import { RepoDto } from 'dogma/features/repo/RepoDto'; import { MirrorDto } from 'dogma/features/project/settings/mirrors/MirrorDto'; import { CredentialDto } from 'dogma/features/project/settings/credentials/CredentialDto'; import { FiBox } from 'react-icons/fi'; +import cronstrue from 'cronstrue'; interface MirrorFormProps { projectName: string; defaultValue: MirrorDto; - onSubmit: ( - credential: MirrorDto, - onSuccess: () => void, - setError: UseFormSetError, - ) => Promise; + onSubmit: (mirror: MirrorDto, onSuccess: () => void, setError: UseFormSetError) => Promise; isWaitingResponse: boolean; } @@ -77,17 +76,20 @@ const MirrorForm = ({ projectName, defaultValue, onSubmit, isWaitingResponse }: const { register, handleSubmit, - reset, - formState: { errors }, + formState: { errors, isDirty }, setError, setValue, control, + watch, } = useForm(); const isNew = defaultValue.id === ''; const { data: repos } = useGetReposQuery(projectName); const { data: credentials } = useGetCredentialsQuery(projectName); + const [isScheduleEnabled, setScheduleEnabled] = useState(defaultValue.schedule != null); + const schedule = watch('schedule'); + const repoOptions: OptionType[] = (repos || []) .filter((repo: RepoDto) => !INTERNAL_REPOS.has(repo.name)) .map((repo: RepoDto) => ({ @@ -111,7 +113,14 @@ const MirrorForm = ({ projectName, defaultValue, onSubmit, isWaitingResponse }: setValue('credentialId', defaultValue.credentialId); setValue('direction', defaultValue.direction); } - }, [defaultValue, setValue, isNew]); + }, [ + isNew, + setValue, + defaultValue.localRepo, + defaultValue.remoteScheme, + defaultValue.credentialId, + defaultValue.direction, + ]); const defaultRemoteScheme: OptionType = defaultValue.remoteScheme ? { value: defaultValue.remoteScheme, label: defaultValue.remoteScheme } @@ -123,7 +132,7 @@ const MirrorForm = ({ projectName, defaultValue, onSubmit, isWaitingResponse }: return (

{ - return onSubmit(mirror, reset, setError); + return onSubmit(mirror, () => {}, setError); })} >
@@ -160,32 +169,66 @@ const MirrorForm = ({ projectName, defaultValue, onSubmit, isWaitingResponse }: - + + { + if (e.target.checked) { + setValue('schedule', defaultValue.schedule, { + shouldDirty: true, + }); + setScheduleEnabled(true); + } else { + setValue('schedule', null, { + shouldDirty: true, + }); + setScheduleEnabled(false); + } + }} + /> - + {isScheduleEnabled ? ( + <> + + {schedule && ( + + {cronstrue.toString(schedule, { verbose: true })} + + )} + + ) : ( + + + Scheduling is disabled. + + )} {errors.schedule ? ( ) : ( - - - Quartz cron expression {' '} - - is used to describe when the mirroring task is supposed to be triggered. - + isScheduleEnabled && ( + + + Quartz cron expression {' '} + + is used to describe when the mirroring task is supposed to be triggered. + + ) )} @@ -417,6 +460,7 @@ const MirrorForm = ({ projectName, defaultValue, onSubmit, isWaitingResponse }: disabled type="submit" colorScheme="green" + isDisabled={!isDirty} isLoading={isWaitingResponse} loadingText="Updating" marginTop="10px" diff --git a/webapp/src/dogma/features/project/settings/mirrors/MirrorList.tsx b/webapp/src/dogma/features/project/settings/mirrors/MirrorList.tsx index ea417861d3..7305902462 100644 --- a/webapp/src/dogma/features/project/settings/mirrors/MirrorList.tsx +++ b/webapp/src/dogma/features/project/settings/mirrors/MirrorList.tsx @@ -2,10 +2,12 @@ import { ColumnDef, createColumnHelper } from '@tanstack/react-table'; import React, { useMemo } from 'react'; import { DataTableClientPagination } from 'dogma/common/components/table/DataTableClientPagination'; import { useGetMirrorsQuery } from 'dogma/features/api/apiSlice'; -import { Badge, Code, Link } from '@chakra-ui/react'; +import { Badge, Button, Code, Link } from '@chakra-ui/react'; import { GoRepo } from 'react-icons/go'; import { LabelledIcon } from 'dogma/common/components/LabelledIcon'; import { MirrorDto } from 'dogma/features/project/settings/mirrors/MirrorDto'; +import { RunMirror } from '../../../mirror/RunMirrorButton'; +import { FaPlay } from 'react-icons/fa'; // eslint-disable-next-line @typescript-eslint/no-unused-vars export type MirrorListProps = { @@ -54,19 +56,43 @@ const MirrorList = ({ projectName }: MirrorListProps) header: 'Direction', }), columnHelper.accessor((row: MirrorDto) => row.schedule, { - cell: (info) => ( - - {info.getValue()} - - ), + cell: (info) => { + return ( + + {info.getValue() || 'disabled'} + + ); + }, header: 'Schedule', }), + columnHelper.accessor((row: MirrorDto) => row.schedule, { + cell: (info) => { + return ( + + {({ isLoading, onToggle }) => ( + + )} + + ); + }, + header: 'Actions', + }), columnHelper.accessor((row: MirrorDto) => row.enabled, { cell: (info) => { if (info.getValue()) { - return Active; + return Enabled; } else { - return Inactive; + return Disabled; } }, header: 'Status', diff --git a/webapp/src/dogma/features/project/settings/mirrors/MirrorView.tsx b/webapp/src/dogma/features/project/settings/mirrors/MirrorView.tsx index 2da0ddf374..8759d634e1 100644 --- a/webapp/src/dogma/features/project/settings/mirrors/MirrorView.tsx +++ b/webapp/src/dogma/features/project/settings/mirrors/MirrorView.tsx @@ -27,6 +27,9 @@ import { VscMirror, VscRepoClone } from 'react-icons/vsc'; import { MirrorDto } from 'dogma/features/project/settings/mirrors/MirrorDto'; import { CredentialDto } from 'dogma/features/project/settings/credentials/CredentialDto'; import { FiBox } from 'react-icons/fi'; +import cronstrue from 'cronstrue'; +import { RunMirror } from '../../../mirror/RunMirrorButton'; +import { FaPlay } from 'react-icons/fa'; const HeadRow = ({ children }: { children: ReactNode }) => ( @@ -76,8 +79,13 @@ const MirrorView = ({ projectName, mirror, credential }: MirrorViewProps) => { - {mirror.schedule} + {mirror.schedule || 'disabled'} + {mirror.schedule && ( + + {cronstrue.toString(mirror.schedule, { verbose: true })} + + )} @@ -151,14 +159,26 @@ const MirrorView = ({ projectName, mirror, credential }: MirrorViewProps) => { -
- - -
+ + {({ isLoading }) => ( + + )} + +
); diff --git a/webapp/src/dogma/features/services/ErrorMessageParser.ts b/webapp/src/dogma/features/services/ErrorMessageParser.ts index e7ff53dbb5..af32b54168 100644 --- a/webapp/src/dogma/features/services/ErrorMessageParser.ts +++ b/webapp/src/dogma/features/services/ErrorMessageParser.ts @@ -8,7 +8,11 @@ class ErrorMessageParser { return object.error; } if (object.data && object.data.message) { - return object.data.message; + let message = object.data.message; + if (object.data.detail) { + message += '\n' + object.data.detail; + } + return message; } if (object.message) { return object.message; diff --git a/webapp/src/dogma/store.ts b/webapp/src/dogma/store.ts index 7a99b32b4a..0673bff42e 100644 --- a/webapp/src/dogma/store.ts +++ b/webapp/src/dogma/store.ts @@ -33,7 +33,10 @@ export function setupStore(preloadedState?: Partial) { preloadedState, // Adding the api middleware enables caching, invalidation, polling, // and other useful features of `rtk-query`. - middleware: (getDefaultMiddleware) => getDefaultMiddleware().concat(apiSlice.middleware), + middleware: (getDefaultMiddleware) => + getDefaultMiddleware({ + serializableCheck: false, + }).concat(apiSlice.middleware), }); } diff --git a/xds/src/main/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesService.java b/xds/src/main/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesService.java index be9b63f5a6..9f35efdf65 100644 --- a/xds/src/main/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesService.java +++ b/xds/src/main/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesService.java @@ -117,7 +117,9 @@ private void validateWatcherAndPush( if (cause != null) { cause = Exceptions.peel(cause); if (cause instanceof IllegalArgumentException || cause instanceof EntryNotFoundException) { - responseObserver.onError(Status.INVALID_ARGUMENT.withCause(cause).asRuntimeException()); + responseObserver.onError(Status.INVALID_ARGUMENT.withCause(cause) + .withDescription(cause.getMessage()) + .asRuntimeException()); } else { responseObserver.onError(Status.INTERNAL.withCause(cause).asRuntimeException()); } diff --git a/xds/src/test/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesServiceTest.java b/xds/src/test/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesServiceTest.java index 7790d3a6ce..2c011e2bd3 100644 --- a/xds/src/test/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesServiceTest.java +++ b/xds/src/test/java/com/linecorp/centraldogma/xds/k8s/v1/XdsKubernetesServiceTest.java @@ -22,6 +22,7 @@ import static com.linecorp.centraldogma.xds.internal.XdsTestUtil.createGroup; import static com.linecorp.centraldogma.xds.internal.XdsTestUtil.endpoint; import static com.linecorp.centraldogma.xds.k8s.v1.XdsKubernetesService.K8S_WATCHERS_DIRECTORY; +import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -179,7 +180,9 @@ void invalidProperty() throws IOException { watcher = watcher(watcherId, "nginx-service", "invalid-credential-id"); response = createWatcher(watcher, watcherId); assertThat(response.status()).isSameAs(HttpStatus.BAD_REQUEST); - assertThat(response.contentUtf8()).contains("failed to find credential 'invalid-credential-id'"); + assertThatJson(response.contentUtf8()) + .node("grpc-code").isEqualTo("INVALID_ARGUMENT") + .node("message").isEqualTo("failed to find credential 'invalid-credential-id' in @xds/meta"); } @Test