From 6299f07d2607656002e3b5754381ec248fdee55e Mon Sep 17 00:00:00 2001 From: chacham Date: Sun, 11 Jul 2021 22:12:04 +0900 Subject: [PATCH] Add basic client supports for YAML --- .../armeria/legacy/LegacyCentralDogma.java | 12 ++++++++ .../client/armeria/ArmeriaCentralDogma.java | 22 ++++++++++---- .../internal/thrift/ChangeConverter.java | 9 ++++++ .../internal/thrift/EntryConverter.java | 2 ++ .../internal/thrift/QueryConverter.java | 4 +++ .../src/main/thrift/CentralDogma.thrift | 3 ++ .../centraldogma/common/DefaultChange.java | 5 ++++ .../linecorp/centraldogma/common/Query.java | 11 +++++++ .../centraldogma/common/QueryType.java | 7 ++++- .../centraldogma/internal/Jackson.java | 29 ++++++++++++++++++- .../centraldogma/internal/SnakeYaml.java | 4 +++ .../linecorp/centraldogma/internal/Util.java | 15 ++++++++++ .../linecorp/centraldogma/it/GetFileTest.java | 20 +++++++++++++ .../converter/ChangesRequestConverter.java | 3 ++ .../storage/repository/RepositoryUtil.java | 4 ++- 15 files changed, 141 insertions(+), 9 deletions(-) diff --git a/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java b/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java index 2b6495a3cf..3e2d6857ec 100644 --- a/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java +++ b/client/java-armeria-legacy/src/main/java/com/linecorp/centraldogma/client/armeria/legacy/LegacyCentralDogma.java @@ -258,6 +258,12 @@ private static Entry toEntry(Query query, Revision normRev, QueryType " (expected: " + queryType + ')'); } return entryAsJson(query, normRev, content); + case IDENTITY_YAML: + if (receivedEntryType != com.linecorp.centraldogma.internal.thrift.EntryType.YAML) { + throw new CentralDogmaException("invalid entry type. entry type: " + receivedEntryType + + " (expected: " + queryType + ')'); + } + return entryAsYaml(query, normRev, content); case IDENTITY: switch (receivedEntryType) { case JSON: @@ -266,6 +272,8 @@ private static Entry toEntry(Query query, Revision normRev, QueryType return entryAsText(query, normRev, content); case DIRECTORY: return unsafeCast(Entry.ofDirectory(normRev, query.path())); + case YAML: + return entryAsYaml(query, normRev, content); } } throw new Error(); // Should never reach here. @@ -283,6 +291,10 @@ private static Entry entryAsText(Query query, Revision normRev, String return unsafeCast(Entry.ofText(normRev, query.path(), content)); } + private static Entry entryAsYaml(Query query, Revision normRev, String content) { + return unsafeCast(Entry.ofYaml(normRev, query.path(), content)); + } + @Override public CompletableFuture>> getFiles(String projectName, String repositoryName, Revision revision, String pathPattern) { diff --git a/client/java-armeria/src/main/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogma.java b/client/java-armeria/src/main/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogma.java index 5c3cbb56f0..f69f6903d5 100644 --- a/client/java-armeria/src/main/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogma.java +++ b/client/java-armeria/src/main/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogma.java @@ -45,6 +45,8 @@ import javax.annotation.Nullable; +import org.yaml.snakeyaml.nodes.Node; + import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -103,6 +105,7 @@ import com.linecorp.centraldogma.common.RevisionNotFoundException; import com.linecorp.centraldogma.common.ShuttingDownException; import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.SnakeYaml; import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.internal.api.v1.WatchTimeout; @@ -1015,6 +1018,8 @@ private static ArrayNode toJson(Iterable> changes) { final Class contentType = c.type().contentType(); if (contentType == JsonNode.class) { changeNode.set("content", (JsonNode) c.content()); + } else if (contentType == Node.class) { + changeNode.put("content", SnakeYaml.serialize((Node) c.content())); } else if (contentType == String.class) { changeNode.put("content", (String) c.content()); } @@ -1068,6 +1073,12 @@ private static Entry toEntry(Revision revision, JsonNode node, QueryType " (expected: " + queryType + ')'); } return entryAsJson(revision, node, entryPath); + case IDENTITY_YAML: + if (receivedEntryType != EntryType.YAML) { + throw new CentralDogmaException("invalid entry type. entry type: " + receivedEntryType + + " (expected: " + queryType + ')'); + } + return entryAsYaml(revision, node, entryPath); case IDENTITY: switch (receivedEntryType) { case JSON: @@ -1100,13 +1111,10 @@ private static Entry entryAsJson(Revision revision, JsonNode node, String private static Entry entryAsYaml(Revision revision, JsonNode node, String entryPath) { final JsonNode content = getField(node, "content"); - final String content0; - if (content.isContainerNode()) { - content0 = content.toString(); - } else { - content0 = content.asText(); + if (content.getNodeType() != JsonNodeType.STRING) { + throw new CentralDogmaException("Found invalid content for YAML entry"); } - return unsafeCast(Entry.ofYaml(revision, entryPath, content0)); + return unsafeCast(Entry.ofYaml(revision, entryPath, SnakeYaml.readTree(content.asText()))); } private static Commit toCommit(JsonNode node) { @@ -1129,6 +1137,8 @@ private static Change toChange(JsonNode node) { switch (type) { case UPSERT_JSON: return unsafeCast(Change.ofJsonUpsert(actualPath, getField(node, "content"))); + case UPSERT_YAML: + return unsafeCast(Change.ofYamlUpsert(actualPath, getField(node, "content").asText())); case UPSERT_TEXT: return unsafeCast(Change.ofTextUpsert(actualPath, getField(node, "content").asText())); case REMOVE: diff --git a/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/ChangeConverter.java b/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/ChangeConverter.java index ea9f213e4c..27594aa964 100644 --- a/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/ChangeConverter.java +++ b/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/ChangeConverter.java @@ -18,11 +18,14 @@ import javax.annotation.Nullable; +import org.yaml.snakeyaml.nodes.Node; + import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Converter; import com.linecorp.centraldogma.common.ChangeFormatException; import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.SnakeYaml; /** * Provides a function converting back and forth between {@link Change} and @@ -56,6 +59,9 @@ protected Change doForward(com.linecorp.centraldogma.common.Change value) { break; case REMOVE: break; + case UPSERT_YAML: + change.setContent(SnakeYaml.serialize((Node) value.content())); + break; } return change; } @@ -78,6 +84,9 @@ protected com.linecorp.centraldogma.common.Change doBackward(Change c) { case APPLY_TEXT_PATCH: return com.linecorp.centraldogma.common.Change.ofTextPatch(c.getPath(), c.getContent()); + case UPSERT_YAML: + return com.linecorp.centraldogma.common.Change.ofYamlUpsert(c.getPath(), + c.getContent()); } throw new Error(); diff --git a/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/EntryConverter.java b/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/EntryConverter.java index ef767327d6..42af670350 100644 --- a/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/EntryConverter.java +++ b/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/EntryConverter.java @@ -86,6 +86,8 @@ public static EntryType convertEntryType(com.linecorp.centraldogma.common.EntryT switch (type) { case JSON: return EntryType.JSON; + case YAML: + return EntryType.YAML; case TEXT: return EntryType.TEXT; case DIRECTORY: diff --git a/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/QueryConverter.java b/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/QueryConverter.java index fc8adeba5f..9593a35183 100644 --- a/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/QueryConverter.java +++ b/common-legacy/src/main/java/com/linecorp/centraldogma/internal/thrift/QueryConverter.java @@ -44,6 +44,8 @@ protected Query doForward(com.linecorp.centraldogma.common.Query query) { return new Query(query.path(), QueryType.IDENTITY_JSON, Collections.emptyList()); case JSON_PATH: return new Query(query.path(), QueryType.JSON_PATH, query.expressions()); + case IDENTITY_YAML: + return new Query(query.path(), QueryType.IDENTITY_YAML, Collections.emptyList()); } throw new Error(); @@ -59,6 +61,8 @@ protected com.linecorp.centraldogma.common.Query doBackward(Query query) { return com.linecorp.centraldogma.common.Query.ofText(query.getPath()); case IDENTITY_JSON: return com.linecorp.centraldogma.common.Query.ofJson(query.getPath()); + case IDENTITY_YAML: + return com.linecorp.centraldogma.common.Query.ofYaml(query.getPath()); case JSON_PATH: return com.linecorp.centraldogma.common.Query.ofJsonPath(query.getPath(), query.getExpressions()); diff --git a/common-legacy/src/main/thrift/CentralDogma.thrift b/common-legacy/src/main/thrift/CentralDogma.thrift index 5bb95410e5..481c56af00 100644 --- a/common-legacy/src/main/thrift/CentralDogma.thrift +++ b/common-legacy/src/main/thrift/CentralDogma.thrift @@ -47,6 +47,7 @@ enum EntryType { JSON = 1, TEXT = 2, DIRECTORY = 3, + YAML = 4, } enum ChangeType { @@ -56,6 +57,7 @@ enum ChangeType { RENAME = 4, APPLY_JSON_PATCH = 5, APPLY_TEXT_PATCH = 6, + UPSERT_YAML = 7, } enum PropertyType { @@ -192,6 +194,7 @@ enum QueryType { JSON_PATH = 2, IDENTITY_TEXT = 3, IDENTITY_JSON = 4, + IDENTITY_YAML = 5, } struct Query { diff --git a/common/src/main/java/com/linecorp/centraldogma/common/DefaultChange.java b/common/src/main/java/com/linecorp/centraldogma/common/DefaultChange.java index 0d205425c9..5bc8d16970 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/DefaultChange.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/DefaultChange.java @@ -18,12 +18,15 @@ import static com.linecorp.centraldogma.internal.Util.validateFilePath; import static com.linecorp.centraldogma.internal.Util.validateJsonFilePath; +import static com.linecorp.centraldogma.internal.Util.validateYamlFilePath; import static java.util.Objects.requireNonNull; import java.util.Objects; import javax.annotation.Nullable; +import org.yaml.snakeyaml.nodes.Node; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; @@ -100,6 +103,8 @@ private static DefaultChange rejectIncompatibleContent(@Nullable JsonNode con if (type.contentType() == JsonNode.class) { validateJsonFilePath(path, "path"); + } else if (type.contentType() == Node.class) { + validateYamlFilePath(path, "path"); } else { validateFilePath(path, "path"); } diff --git a/common/src/main/java/com/linecorp/centraldogma/common/Query.java b/common/src/main/java/com/linecorp/centraldogma/common/Query.java index 6ec6b80615..306bf47c72 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/Query.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/Query.java @@ -23,6 +23,8 @@ import javax.annotation.Nullable; +import org.yaml.snakeyaml.nodes.Node; + import com.fasterxml.jackson.databind.JsonNode; /** @@ -86,6 +88,15 @@ static Query ofJsonPath(String path, Iterable jsonPaths) { return new JsonPathQuery(path, jsonPaths); } + /** + * Returns a newly-created {@link Query} that retrieves the YAML content as it is. + * + * @param path the path of a file being queried on + */ + static Query ofYaml(String path) { + return new IdentityQuery<>(path, QueryType.IDENTITY_YAML); + } + /** * Returns a newly-created {@link Query} that applies a series of expressions to the content. * diff --git a/common/src/main/java/com/linecorp/centraldogma/common/QueryType.java b/common/src/main/java/com/linecorp/centraldogma/common/QueryType.java index 95518af47f..7d3cf38b0b 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/QueryType.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/QueryType.java @@ -42,7 +42,12 @@ public enum QueryType { * Applies a series of JSON path * expressions to the content. */ - JSON_PATH(EnumSet.of(EntryType.JSON)); + JSON_PATH(EnumSet.of(EntryType.JSON)), + + /** + * Retrieves the YAML content as it is. + */ + IDENTITY_YAML(EnumSet.of(EntryType.YAML)); private final Set supportedEntryTypes; diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/Jackson.java b/common/src/main/java/com/linecorp/centraldogma/internal/Jackson.java index d9ed1743c0..d417c6d602 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/Jackson.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/Jackson.java @@ -28,9 +28,12 @@ import java.util.Iterator; import java.util.Set; +import org.yaml.snakeyaml.nodes.Node; + import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.core.io.JsonStringEncoder; @@ -38,11 +41,15 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.util.DefaultIndenter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.node.JsonNodeType; @@ -78,7 +85,9 @@ public final class Jackson { prettyMapper.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); registerModules(new SimpleModule().addSerializer(Instant.class, InstantSerializer.INSTANCE) - .addDeserializer(Instant.class, InstantDeserializer.INSTANT)); + .addDeserializer(Instant.class, InstantDeserializer.INSTANT) + .addSerializer(Node.class, new YamlSerializer()) + .addDeserializer(Node.class, new YamlDeserializer())); } private static final JsonFactory compactFactory = new JsonFactory(compactMapper); @@ -382,4 +391,22 @@ private static class PrettyPrinterImpl extends DefaultPrettyPrinter { _objectIndenter = objectIndenter; } } + + private static class YamlSerializer extends JsonSerializer { + + @Override + public void serialize(Node value, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + gen.writeString(SnakeYaml.serialize(value)); + } + } + + private static class YamlDeserializer extends JsonDeserializer { + + @Override + public Node deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + return SnakeYaml.readTree(p.getValueAsString()); + } + } } diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/SnakeYaml.java b/common/src/main/java/com/linecorp/centraldogma/internal/SnakeYaml.java index 70a03fa83d..879d54da41 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/SnakeYaml.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/SnakeYaml.java @@ -47,5 +47,9 @@ public static String serialize(Node data) { return stringWriter.toString(); } + public static String dump(Object o) { + return yaml.dump(o); + } + private SnakeYaml() {} } diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/Util.java b/common/src/main/java/com/linecorp/centraldogma/internal/Util.java index 110e9b1933..9a9f774585 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/Util.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/Util.java @@ -43,6 +43,8 @@ public final class Util { "^(?:/[-_0-9a-zA-Z](?:[-_.0-9a-zA-Z]*[-_0-9a-zA-Z])?)+$"); private static final Pattern JSON_FILE_PATH_PATTERN = Pattern.compile( "^(?:/[-_0-9a-zA-Z](?:[-_.0-9a-zA-Z]*[-_0-9a-zA-Z])?)+\\.(?i)json$"); + private static final Pattern YAML_FILE_PATH_PATTERN = Pattern.compile( + "^(?:/[-_0-9a-zA-Z](?:[-_.0-9a-zA-Z]*[-_0-9a-zA-Z])?)+\\.(?i)ya?ml$"); private static final Pattern DIR_PATH_PATTERN = Pattern.compile( "^(?:/[-_0-9a-zA-Z](?:[-_.0-9a-zA-Z]*[-_0-9a-zA-Z])?)*/?$"); private static final Pattern PATH_PATTERN_PATTERN = Pattern.compile("^[- /*_.,0-9a-zA-Z]+$"); @@ -113,6 +115,19 @@ public static boolean isValidJsonPath(String jsonPath) { } } + public static String validateYamlFilePath(String path, String paramName) { + requireNonNull(path, paramName); + checkArgument(isValidYamlFilePath(path), + "%s: %s (expected: %s)", paramName, path, YAML_FILE_PATH_PATTERN); + return path; + } + + public static boolean isValidYamlFilePath(String path) { + requireNonNull(path, "path"); + return !path.isEmpty() && path.charAt(0) == '/' && + YAML_FILE_PATH_PATTERN.matcher(path).matches(); + } + public static String validateDirPath(String path, String paramName) { requireNonNull(path, paramName); checkArgument(isValidDirPath(path), diff --git a/it/src/test/java/com/linecorp/centraldogma/it/GetFileTest.java b/it/src/test/java/com/linecorp/centraldogma/it/GetFileTest.java index 102a05fb35..9e2fea027e 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/GetFileTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/GetFileTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.yaml.snakeyaml.nodes.Node; import com.fasterxml.jackson.databind.JsonNode; @@ -38,6 +39,7 @@ import com.linecorp.centraldogma.common.QueryExecutionException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.SnakeYaml; class GetFileTest { @@ -61,6 +63,24 @@ void getJsonAsText(ClientType clientType) throws Exception { Change.ofRemoval("/test/foo.json")).join(); } + @ParameterizedTest + @EnumSource(ClientType.class) + void getYamlAsText(ClientType clientType) throws Exception { + final CentralDogma client = clientType.client(dogma); + client.push(dogma.project(), dogma.repo1(), Revision.HEAD, "Add a file", + Change.ofYamlUpsert("/test/foo.yaml", "a.b: \"c\"")).join(); + final Entry yaml = client.getFile(dogma.project(), dogma.repo1(), Revision.HEAD, + Query.ofYaml("/test/foo.yaml")).join(); + assertThat(SnakeYaml.serialize(yaml.content())) + .isEqualTo(SnakeYaml.serialize(SnakeYaml.readTree("a.b: \"c\""))); + + final Entry text = client.getFile(dogma.project(), dogma.repo1(), Revision.HEAD, + Query.ofText("/test/foo.yaml")).join(); + assertThat(text.content()).isEqualTo(SnakeYaml.serialize(SnakeYaml.readTree("a.b: \"c\""))); + client.push(dogma.project(), dogma.repo1(), Revision.HEAD, "Remove a file", + Change.ofRemoval("/test/foo.yaml")).join(); + } + @ParameterizedTest @EnumSource(ClientType.class) void invalidJsonPath(ClientType clientType) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/ChangesRequestConverter.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/ChangesRequestConverter.java index f8095da539..85c0ee6382 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/ChangesRequestConverter.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/ChangesRequestConverter.java @@ -91,6 +91,9 @@ private static Change readChange(JsonNode node) { if (changeType == ChangeType.UPSERT_JSON) { return Change.ofJsonUpsert(path, node.get("content")); } + if (changeType == ChangeType.UPSERT_YAML) { + return Change.ofYamlUpsert(path, node.get("content").textValue()); + } if (changeType == ChangeType.REMOVE) { return Change.ofRemoval(path); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryUtil.java index 575fdf7e06..a3f0427d22 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryUtil.java @@ -20,6 +20,7 @@ import static com.linecorp.centraldogma.common.QueryType.IDENTITY; import static com.linecorp.centraldogma.common.QueryType.IDENTITY_JSON; import static com.linecorp.centraldogma.common.QueryType.IDENTITY_TEXT; +import static com.linecorp.centraldogma.common.QueryType.IDENTITY_YAML; import static com.linecorp.centraldogma.common.QueryType.JSON_PATH; import static com.linecorp.centraldogma.internal.Util.unsafeCast; import static java.util.Objects.requireNonNull; @@ -138,7 +139,8 @@ static Entry applyQuery(Entry entry, Query query) { " (query: " + query + ')'); } - if (queryType == IDENTITY || queryType == IDENTITY_TEXT || queryType == IDENTITY_JSON) { + if (queryType == IDENTITY || queryType == IDENTITY_TEXT || queryType == IDENTITY_JSON || + queryType == IDENTITY_YAML) { return entry; } else if (queryType == JSON_PATH) { return Entry.of(entry.revision(), query.path(), entryType, query.apply(entry.content()));