Tree-sitter

.gitattributes

@@ -0,0 +1 @@
+modules/treesitter/src/main/scala/playground/generated/** linguist-generated

.github/workflows/ci.yml

@@ -57,3 +57,20 @@ jobs:
       - name: Show extension test logs
         if: always() && job.status == 'failure'
         run: cat vscode-extension/fixture/smithyql-log.txt | tail --lines 1000
+
+  build-parser:
+    name: "Build parser"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, macos-13]
+    runs-on: ${{matrix.os}}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/[email protected]
+      - uses: coursier/setup-action@v1
+        with:
+          apps: sbt
+          jvm: adoptium:1.21
+      - name: Parser tests
+        # intentionally not setting up nix
+        run: sbt treesitter/test

.gitignore

@@ -10,3 +10,4 @@ smithyql-log.txt
 
 result
 .version
+build/smithy

build.sbt

@@ -87,6 +87,7 @@ val commonSettings = Seq(
   Test / scalacOptions += "-Wconf:cat=deprecation:silent,msg=Specify both message and version:silent",
   scalacOptions += "-release:11",
   mimaFailOnNoPrevious := false,
+  resolvers += "Sonatype S01 snapshots" at "https://s01.oss.sonatype.org/content/repositories/snapshots",
 )
 
 def module(
@@ -133,8 +134,29 @@ lazy val parser = module("parser")
   .dependsOn(
     ast % "test->test;compile->compile",
     source % "test->test;compile->compile",
+    treesitter % "test->compile",
   )
 
+lazy val treesitter = module("treesitter")
+  .settings(
+    libraryDependencies ++= Seq(
+      "org.polyvariant.treesitter4s" %% "core" % "0.4.0"
+    )
+  )
+
+lazy val parsergen = module("parser-gen")
+  .settings(
+    libraryDependencies ++= Seq(
+      "dev.optics" %% "monocle-core" % "3.3.0",
+      "com.disneystreaming.smithy4s" %% "smithy4s-json" % smithy4sVersion.value,
+      ("org.scalameta" %% "scalameta" % "4.11.0").cross(CrossVersion.for3Use2_13),
+      "org.polyvariant.treesitter4s" %% "core" % "0.4.0",
+      "com.lihaoyi" %% "os-lib" % "0.11.3",
+    ),
+    scalacOptions -= "-release:11",
+  )
+  .enablePlugins(Smithy4sCodegenPlugin)
+
 // Formatter for the SmithyQL language constructs
 lazy val formatter = module("formatter")
   .settings(
@@ -189,6 +211,7 @@ lazy val core = module("core")
     examples % "test->compile",
     pluginCore,
     ast,
+    treesitter,
     source % "test->test;compile->compile",
     parser % "test->compile;test->test",
     formatter,
@@ -239,6 +262,7 @@ lazy val e2e = module("e2e")
             parser / publishLocal,
             pluginCore / publishLocal,
             source / publishLocal,
+            treesitter / publishLocal,
             ast / publishLocal,
             formatter / publishLocal,
             protocol4s / publishLocal,
@@ -269,11 +293,13 @@ lazy val root = project
     core,
     examples,
     parser,
+    parsergen,
     formatter,
     languageSupport,
     lsp,
     protocol4s,
     pluginCore,
     pluginSample,
     e2e,
+    treesitter,
   )

flake.nix

@@ -1,27 +1,47 @@
 {
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
     flake-utils.url = "github:numtide/flake-utils";
   };
 
   outputs = { self, nixpkgs, flake-utils, ... }:
-    flake-utils.lib.eachDefaultSystem (
-      system:
-      let
-        pkgs = import nixpkgs { inherit system; };
-      in
-      {
+    flake-utils.lib.eachDefaultSystem (system:
+      let pkgs = import nixpkgs { inherit system; };
+      in {
         devShells.default = pkgs.mkShell {
           buildInputs = [
             pkgs.yarn
             pkgs.nodejs
             pkgs.sbt
             pkgs.jless
             pkgs.gnupg
+            (pkgs.tree-sitter.override { webUISupport = true; })
             # temporary, while we don't download coursier ourselves
             pkgs.coursier
           ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.xvfb-run ];
         };
-      }
-    );
+        packages.tree-sitter-smithyql = pkgs.stdenv.mkDerivation {
+          name = "tree-sitter-smithyql";
+          src = ./tree-sitter-smithyql;
+          buildInputs = [ pkgs.tree-sitter pkgs.nodejs ];
+          buildPhase = ''
+            tree-sitter generate
+            cc src/parser.c -shared -o $out
+          '';
+          dontInstall = true;
+        };
+        packages.tree-sitter-smithyql-all = pkgs.stdenv.mkDerivation {
+          name = "tree-sitter-smithyql-all";
+          src = ./tree-sitter-smithyql;
+          dontBuild = true;
+          installPhase = ''
+            mkdir $out
+            cd $out
+            mkdir darwin-aarch64 && cp ${self.packages.aarch64-darwin.tree-sitter-smithyql} darwin-aarch64/libtree-sitter-smithyql.dylib
+            mkdir darwin-x86-64 && cp ${self.packages.x86_64-darwin.tree-sitter-smithyql} darwin-x86-64/libtree-sitter-smithyql.dylib
+            mkdir linux-aarch64 && cp ${self.packages.aarch64-linux.tree-sitter-smithyql} linux-aarch64/libtree-sitter-smithyql.so
+            mkdir linux-x86-64 && cp ${self.packages.x86_64-linux.tree-sitter-smithyql} linux-x86-64/libtree-sitter-smithyql.so
+          '';
+        };
+      });
 }

modules/core/src/main/scala/playground/ASTAdapter.scala

@@ -0,0 +1,12 @@
+package playground
+
+import cats.syntax.all.*
+import playground.smithyql.QualifiedIdentifier
+
+object ASTAdapter {
+
+  def decodeQI(qi: playground.generated.nodes.QualifiedIdentifier): Option[QualifiedIdentifier] =
+    (qi.namespace.map(_.source).toNel, qi.selection.map(_.source))
+      .mapN(QualifiedIdentifier.apply)
+
+}

modules/core/src/main/scala/playground/MultiServiceResolver.scala

@@ -9,6 +9,7 @@ import playground.smithyql.UseClause
 import playground.smithyql.WithSource
 
 object MultiServiceResolver {
+  import playground.smithyql.tsutils.*
 
   /** Determines which service should be used for a query. The rules are:
     *   - If the operation name has a service identifier, there MUST be a service with that name
@@ -37,6 +38,34 @@ object MultiServiceResolver {
       case None => resolveImplicit(queryOperationName.operationName, serviceIndex, useClauses)
     }
 
+  /** Determines which service should be used for a query. The rules are:
+    *   - If the operation name has a service identifier, there MUST be a service with that name
+    *     that contains the given operation.
+    *   - If there's no service identifier, find all matching services that are included in the use
+    *     clauses. MUST find exactly one entry.
+    *
+    * In other cases, such as when we can't find a unique entry, or the explicitly referenced
+    * service doesn't have an operation with a matching name, we fail. The latter might eventually
+    * be refactored to a separate piece of code.
+    *
+    * **Important**!
+    *
+    * This method assumes that all of the use clauses match the available service set. It does NOT
+    * perform a check on that. For the actual check, see PreludeCompiler.
+    */
+  def resolveServiceTs(
+    queryOperationName: playground.generated.nodes.QueryOperationName,
+    serviceIndex: ServiceIndex,
+    useClauses: List[playground.generated.nodes.UseClause],
+  ): EitherNel[CompilationError, Option[QualifiedIdentifier]] = queryOperationName
+    .name
+    .flatTraverse { opName =>
+      queryOperationName.service_identifier match {
+        case Some(explicitRef) => resolveExplicitTs(serviceIndex, explicitRef, opName)
+        case None              => resolveImplicitTs(opName, serviceIndex, useClauses).map(_.some)
+      }
+    }
+
   private def resolveExplicit(
     index: ServiceIndex,
     explicitRef: WithSource[QualifiedIdentifier],
@@ -66,6 +95,39 @@ object MultiServiceResolver {
       case Some(_) => explicitRef.value.asRight
     }
 
+  private def resolveExplicitTs(
+    index: ServiceIndex,
+    explicitRef: playground.generated.nodes.QualifiedIdentifier,
+    operationName: playground.generated.nodes.OperationName,
+  ): EitherNel[CompilationError, Option[QualifiedIdentifier]] = ASTAdapter
+    .decodeQI(explicitRef)
+    .traverse { ref =>
+      index.getService(ref) match {
+        // explicit reference exists, but the service doesn't
+        case None =>
+          CompilationError
+            .error(
+              CompilationErrorDetails.UnknownService(index.serviceIds.toList),
+              explicitRef.range,
+            )
+            .leftNel
+
+        // the service exists, but doesn't have the requested operation
+        case Some(service)
+            if !service.operationNames.contains_(OperationName(operationName.source)) =>
+          CompilationError
+            .error(
+              CompilationErrorDetails.OperationMissing(service.operationNames.toList),
+              operationName.range,
+            )
+            .leftNel
+
+        // all good
+        case Some(_) => ref.asRight
+      }
+
+    }
+
   private def resolveImplicit(
     operationName: WithSource[OperationName[WithSource]],
     index: ServiceIndex,
@@ -90,4 +152,28 @@ object MultiServiceResolver {
     }
   }
 
+  private def resolveImplicitTs(
+    operationName: playground.generated.nodes.OperationName,
+    index: ServiceIndex,
+    useClauses: List[playground.generated.nodes.UseClause],
+  ): EitherNel[CompilationError, QualifiedIdentifier] = {
+    val matchingServices = index
+      .getServices(useClauses.flatMap(_.identifier).flatMap(ASTAdapter.decodeQI).toSet)
+      .filter(_.hasOperation(OperationName(operationName.source)))
+
+    matchingServices match {
+      case one :: Nil => one.id.asRight
+      case _ =>
+        CompilationError
+          .error(
+            CompilationErrorDetails
+              .AmbiguousService(
+                workspaceServices = index.serviceIds.toList
+              ),
+            operationName.range,
+          )
+          .leftNel
+    }
+  }
+
 }