-
Notifications
You must be signed in to change notification settings - Fork 94
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[NU-1735] Use installation example for a local development purposes (#…
- Loading branch information
1 parent
825d320
commit 95e6043
Showing
23 changed files
with
8,648 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| <component name="ProjectRunConfigurationManager"> | ||
| <configuration default="false" name="RunEnvForLocalDesigner" type="Application" factoryName="Application"> | ||
| <option name="MAIN_CLASS_NAME" value="pl.touk.nussknacker.dev.RunEnvForLocalDesigner" /> | ||
| <module name="nussknacker-designer" /> | ||
| <extension name="coverage"> | ||
| <pattern> | ||
| <option name="PATTERN" value="pl.touk.nussknacker.dev.*" /> | ||
| <option name="ENABLED" value="true" /> | ||
| </pattern> | ||
| </extension> | ||
| <method v="2"> | ||
| <option name="Make" enabled="true" /> | ||
| </method> | ||
| </configuration> | ||
| </component> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
99 changes: 99 additions & 0 deletions
99
designer/server/src/test/scala/pl/touk/nussknacker/dev/RunEnvForLocalDesigner.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| package pl.touk.nussknacker.dev | ||
|
|
||
| import cats.effect.{ExitCode, IO, IOApp} | ||
| import com.dimafeng.testcontainers.{DockerComposeContainer, WaitingForService} | ||
| import com.typesafe.scalalogging.LazyLogging | ||
| import org.testcontainers.containers.wait.strategy.ShellStrategy | ||
| import pl.touk.nussknacker.dev.RunEnvForLocalDesigner.Config.ScalaV | ||
| import scopt.{OParser, Read} | ||
|
|
||
| import java.io.{File => JFile} | ||
|
|
||
| // You can use it for a development purposes. It runs docker compose defined in `examples/dev` folder. | ||
| // After running this class you can run Nu Designer locally that can connect to the exposed services. | ||
| object RunEnvForLocalDesigner extends IOApp with LazyLogging { | ||
|
|
||
| override def run(args: List[String]): IO[ExitCode] = for { | ||
| config <- readConfig(args) | ||
| _ <- log(s"Starting docker compose-based stack (for ${config.scalaV}) to be used with locally run Nu Designer...") | ||
| _ <- createDockerEnv(config) | ||
| _ <- log("You can run designer now...") | ||
| _ <- IO.never[Unit] | ||
| } yield ExitCode.Success | ||
|
|
||
| private def readConfig(args: List[String]) = IO.delay { | ||
| OParser | ||
| .parse(Config.parser, args, Config()) | ||
| .getOrElse(throw new Exception("Invalid arguments")) | ||
| } | ||
|
|
||
| private def createDockerEnv(config: Config) = IO.delay { | ||
| val scalaVOverrideYmlFile = config.scalaV match { | ||
| case ScalaV.Scala212 => None | ||
| case ScalaV.Scala213 => Some(new JFile("examples/dev/nu-scala213.override.yml")) | ||
| } | ||
| val env = new LocalTestingEnvDockerCompose( | ||
| dockerComposeTweakFiles = scalaVOverrideYmlFile.toList ::: config.customizeYaml.toList | ||
| ) | ||
| env.start() | ||
| env | ||
| } | ||
|
|
||
| private def log(message: => String) = IO.delay(logger.info(message)) | ||
|
|
||
| final case class Config(scalaV: ScalaV = ScalaV.Scala213, customizeYaml: Option[JFile] = None) | ||
|
|
||
| object Config { | ||
|
|
||
| sealed trait ScalaV | ||
|
|
||
| object ScalaV { | ||
| case object Scala212 extends ScalaV | ||
| case object Scala213 extends ScalaV | ||
|
|
||
| implicit val scalaVRead: Read[ScalaV] = | ||
| scopt.Read.reads(_.toLowerCase).map { | ||
| case "scala212" => ScalaV.Scala212 | ||
| case "scala213" => ScalaV.Scala213 | ||
| } | ||
|
|
||
| } | ||
|
|
||
| private val builder = OParser.builder[Config] | ||
|
|
||
| import builder._ | ||
|
|
||
| lazy val parser: OParser[Unit, Config] = OParser.sequence( | ||
| head("Env for local development of Nu Designer"), | ||
| programName("sbt designer/test:runMain pl.touk.nussknacker.dev.RunEnvForLocalDesigner"), | ||
| opt[ScalaV]('s', "scalaV") | ||
| .optional() | ||
| .action((scalaV, c) => c.copy(scalaV = scalaV)) | ||
| .text("Scala version. Available options: scala212, scala213"), | ||
| opt[JFile]('c', "customizeYaml") | ||
| .optional() | ||
| .valueName("<absolute file path>") | ||
| .validate { file => | ||
| if (!file.exists()) Left(s"'$file' does NOT exist") | ||
| else if (!file.isFile) Left(s"'$file' is NOT a file") | ||
| else if (!file.canRead) Left(s"CANNOT read the file '$file'") | ||
| else Right(()) | ||
| } | ||
| .action((customizeYaml, c) => c.copy(customizeYaml = Some(customizeYaml))) | ||
| .text("Yaml file for docker compose override"), | ||
| ) | ||
|
|
||
| } | ||
|
|
||
| class LocalTestingEnvDockerCompose(dockerComposeTweakFiles: Iterable[JFile]) | ||
| extends DockerComposeContainer( | ||
| composeFiles = new JFile("examples/dev/local-testing.docker-compose.yml") :: | ||
| dockerComposeTweakFiles.toList, | ||
| waitingFor = Some( | ||
| WaitingForService("wait-for-all", new ShellStrategy().withCommand("pwd")), | ||
| ), | ||
| // Change to 'true' to enable logging | ||
| tailChildContainers = false | ||
| ) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| ARG FLINK_VERSION | ||
|
|
||
| FROM flink:${FLINK_VERSION} | ||
|
|
||
| USER root | ||
| RUN echo '#!/bin/sh' > /ex-docker-entrypoint.sh && \ | ||
| echo 'export FLINK_PROPERTIES=$(cat /opt/flink/conf/flink-properties.yml) && /docker-entrypoint.sh "$@"' >> /ex-docker-entrypoint.sh && \ | ||
| chmod +x /ex-docker-entrypoint.sh | ||
|
|
||
| USER flink | ||
| COPY flink-properties.yml /opt/flink/conf/ | ||
| RUN mkdir -p /opt/flink/data && \ | ||
| chmod -R 777 /opt/flink/data | ||
|
|
||
| VOLUME /opt/flink/data | ||
|
|
||
| ENTRYPOINT [ "/ex-docker-entrypoint.sh" ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| taskmanager.numberOfTaskSlots: 8 | ||
|
|
||
| state.backend: filesystem | ||
| state.checkpoints.dir: file:///opt/flink/data/checkpoints | ||
| state.savepoints.dir: file:///opt/flink/data/savepoints | ||
|
|
||
| #Below are base settings for rocksdb metrics, that can be used for grafana dashboards | ||
| state.backend.rocksdb.metrics.estimate-num-keys: true | ||
| state.backend.rocksdb.metrics.estimate-live-data-size: true | ||
| state.backend.rocksdb.metrics.cur-size-all-mem-tables: true | ||
| state.backend.rocksdb.metrics.size-all-mem-tables: true | ||
| # We can have many jobs per cluster, in such setting managed memory is not easy to tune | ||
| state.backend.rocksdb.memory.managed: false | ||
| # For frequent writes increase the value as needed. Currently RocksDB settings can only be changed per Flink cluster | ||
| state.backend.rocksdb.writebuffer.size: 256m | ||
|
|
||
| metrics.reporters: influxdb_reporter | ||
| metrics.reporter.influxdb_reporter.factory.class: org.apache.flink.metrics.influxdb.InfluxdbReporterFactory | ||
| metrics.reporter.influxdb_reporter.host: telegraf | ||
| metrics.reporter.influxdb_reporter.port: 8087 | ||
| metrics.reporter.influxdb_reporter.db: nussknacker_metrics | ||
| metrics.reporter.influxdb_reporter.scope.variables.excludes: tm_id;job_id;task_id;task_attempt_id;operator_id;task_attempt_num;task_name | ||
| metrics.scope.jm: local.<host>.jobmanagerGlobal | ||
| metrics.scope.jm.job: local.<host>.jobmanagerJob.<job_name> | ||
| metrics.scope.tm: local.<host>.taskmanagerGlobal.<tm_id> | ||
| metrics.scope.tm.job: local.<host>.taskmanagerJob.<tm_id>.<job_name> | ||
| metrics.scope.task: local.<host>.taskmanagerTask.<tm_id>.<job_name>.<task_name>.<subtask_index> | ||
| metrics.scope.operator: local.<host>.taskmanagerTask.<tm_id>.<job_name>.<operator_name>.<subtask_index> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| apiVersion: 1 | ||
| providers: | ||
| - name: default | ||
| orgId: 1 | ||
| folder: '' | ||
| folderUid: '' | ||
| type: file | ||
| disableDeletion: true | ||
| editable: true | ||
| updateIntervalSeconds: 10 | ||
| allowUiUpdates: true | ||
| options: | ||
| path: /var/lib/grafana/dashboards |
Oops, something went wrong.