diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7e30eb8..9b6e40e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -62,6 +62,7 @@ jobs: - run: npm run -w @holoom/types build - run: npm run -w @holoom/client build - run: npm run -w @holoom/authority build + - run: npm run -w @holoom/sandbox build - uses: actions/upload-artifact@v4 with: @@ -78,6 +79,11 @@ jobs: name: authority-dist path: packages/authority/dist retention-days: 1 + - uses: actions/upload-artifact@v4 + with: + name: sandbox-dist + path: packages/sandbox/dist + retention-days: 1 tryorama-tests: name: Tryorama Tests @@ -148,21 +154,22 @@ jobs: - name: Move DNA run: mv happ_workdir/holoom.dna ./holoom.dna - - name: Download types dist - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: name: types-dist path: packages/types/dist - - name: Download client dist - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: name: client-dist path: packages/client/dist - - name: Download external-id-attestor dist - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: name: authority-dist path: packages/authority/dist + - uses: actions/download-artifact@v4 + with: + name: sandbox-dist + path: packages/sandbox/dist - name: Release binaries uses: softprops/action-gh-release@v2 @@ -185,3 +192,7 @@ jobs: run: npm publish --access public -w @holoom/authority env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Publish sandbox to npm + run: npm publish --access public -w @holoom/sandbox + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/examples/emissions/co2-sensor/Dockerfile b/examples/emissions/co2-sensor/Dockerfile new file mode 100644 index 0000000..a0a86d9 --- /dev/null +++ b/examples/emissions/co2-sensor/Dockerfile @@ -0,0 +1,36 @@ +# We use ubuntu as it's glibc version is compatible with the prebuilt binaries +FROM ubuntu + +RUN apt-get update && apt-get install -y wget + +# Install node v20.12.2 +ENV NVM_DIR /usr/local/nvm +ENV NODE_VERSION v20.12.2 +RUN mkdir -p $NVM_DIR && wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash +RUN /bin/bash -c "source $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use --delete-prefix $NODE_VERSION" +ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/bin +ENV PATH $NODE_PATH:$PATH + +# Install prebuilt holochain binaries +RUN wget -nv https://github.com/holochain/holochain/releases/download/holochain-0.4.0-dev.20/hc-x86_64-linux \ + -O /usr/local/bin/hc +RUN wget -nv https://github.com/holochain/holochain/releases/download/holochain-0.4.0-dev.20/holochain-x86_64-linux \ + -O /usr/local/bin/holochain +RUN wget -nv https://github.com/holochain/holochain/releases/download/holochain-0.4.0-dev.20/lair-keystore-x86_64-linux \ + -O /usr/local/bin/lair-keystore +RUN chmod 755 /usr/local/bin/hc /usr/local/bin/holochain /usr/local/bin/lair-keystore + +# Install tsx +WORKDIR /home/node +RUN /bin/bash -c "source $NVM_DIR/nvm.sh && npm i tsx" + +# Copy the actual server script +COPY ./co2-sensor.ts ./co2-sensor.ts + +# So container runs with nvm loaded +SHELL ["/bin/bash", "--login", "-c"] + +# The ./packages directory is mounted from the locally built npm workspace. We +# npm install at launch to avoid the need for the package.json to know the +# latest version or features of the workspace. +CMD npm install ./packages/types ./packages/sandbox; npx tsx ./co2-sensor.ts diff --git a/examples/emissions/co2-sensor/co2-sensor.ts b/examples/emissions/co2-sensor/co2-sensor.ts new file mode 100644 index 0000000..99f4421 --- /dev/null +++ b/examples/emissions/co2-sensor/co2-sensor.ts @@ -0,0 +1,47 @@ +/** + * A mock CO₂ sensor, that emits a random measure in grams roughly every 2 + * seconds. The "measurements" are published by a sandboxed holochain agent + * that serves no other role in the network. + */ + +import { ensureAndConnectToHapp } from "@holoom/sandbox"; +import { UsernameRegistryCoordinator } from "@holoom/types"; + +async function main() { + // Create a conductor sandbox (with holoom installed) at the specified + // directory if it doesn't already exist, and connect to it. + const { appWs } = await ensureAndConnectToHapp( + "/sandbox", + "/workdir/holoom.happ", + "emissions-local-test-2024-09-04T12:59", + { + bootstrapServerUrl: new URL("https://bootstrap-0.infra.holochain.org"), + signalingServerUrl: new URL("wss://sbd-0.main.infra.holo.host"), + password: "password", + } + ); + const usernameRegistryCoordinator = new UsernameRegistryCoordinator(appWs); + + setInterval(() => publishMeasurement(usernameRegistryCoordinator), 1_000); +} + +async function publishMeasurement( + usernameRegistryCoordinator: UsernameRegistryCoordinator +) { + const time = Math.floor(Date.now() / 1000); + const name = `co2-sensor/time/${time}`; + + // A pretend measure of detected CO₂ + const gramsCo2 = Math.floor(Math.random() * 1000); + try { + await usernameRegistryCoordinator.createOracleDocument({ + name, + json_data: JSON.stringify({ time, gramsCo2 }), + }); + console.log("Published", time, gramsCo2); + } catch (err) { + console.error("Failed to publish reading with error:", err); + } +} + +main(); diff --git a/examples/emissions/docker-compose.yaml b/examples/emissions/docker-compose.yaml new file mode 100644 index 0000000..66b718e --- /dev/null +++ b/examples/emissions/docker-compose.yaml @@ -0,0 +1,7 @@ +services: + + co2-sensor: + build: ./co2-sensor + volumes: + - ../../packages:/home/node/packages + - ../../workdir:/workdir \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 6b71608..35c9eec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4126,9 +4126,9 @@ } }, "node_modules/yaml": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.0.tgz", - "integrity": "sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.1.tgz", + "integrity": "sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==", "bin": { "yaml": "bin.mjs" }, @@ -4147,7 +4147,8 @@ "bs58": "^4.0.1", "dotenv": "^16.4.5", "express": "^4.19.2", - "viem": "^2.8.13" + "viem": "^2.8.13", + "yaml": "^2.5.1" }, "devDependencies": { "@rollup/plugin-alias": "^5.1.0", diff --git a/package.json b/package.json index 5f91618..94ff52b 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,10 @@ }, "scripts": { "build:dna": "scripts/build_dna.sh", - "test:tryorama": "npm run build:dna && npm run prepare-build:types && npm run -w @holoom/client -w @holoom/authority build && npm t -w @holoom/tryorama", - "test:dna": "npm run build:dna && cargo nextest run -j 1", - "prepare-build:types": "rimraf crates/holoom_types/bindings && cargo test --package holoom_types && npm run -w @holoom/types prepare:bindings && npm run -w @holoom/types build", - "build:client": "npm run build -w @holoom/client", - "build:authority": "npm run build -w @holoom/authority", + "test:tryorama": "npm run build:dna && npm run prepare:types && npm run build:packages && npm t -w @holoom/tryorama", + "prepare:types": "rimraf crates/holoom_types/bindings && cargo test --package holoom_types && npm run -w @holoom/types prepare:bindings", + "build:packages": "npm run -w @holoom/types -w @holoom/client -w @holoom/authority -w @holoom/sandbox build", + "example:emissions": "npm run build:packages && cd examples/emissions && docker-compose build && docker-compose up", "typedoc": "typedoc --options packages/typedoc.json" }, "devDependencies": { diff --git a/packages/authority/tsconfig.json b/packages/authority/tsconfig.json index 5470e60..aaa9cce 100644 --- a/packages/authority/tsconfig.json +++ b/packages/authority/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "../../tsconfig.json", "include": ["src", "rollup.node.config.ts", "rollup.browser.config.ts"] -} \ No newline at end of file +} diff --git a/packages/sandbox/LICENSE b/packages/sandbox/LICENSE new file mode 100644 index 0000000..9cf1062 --- /dev/null +++ b/packages/sandbox/LICENSE @@ -0,0 +1,19 @@ +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md new file mode 100644 index 0000000..a6bd9cc --- /dev/null +++ b/packages/sandbox/README.md @@ -0,0 +1,3 @@ +# @holoom/authority + +Helpers for creating and connecting to a holochain sandbox with holoom installed. diff --git a/packages/sandbox/build.tsconfig.json b/packages/sandbox/build.tsconfig.json new file mode 100644 index 0000000..614f4ec --- /dev/null +++ b/packages/sandbox/build.tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "declarationDir": "dist/types" + }, + "include": ["src"] +} \ No newline at end of file diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json new file mode 100644 index 0000000..471f776 --- /dev/null +++ b/packages/sandbox/package.json @@ -0,0 +1,48 @@ +{ + "name": "@holoom/sandbox", + "version": "0.1.0-dev.13", + "description": "Helpers for starting a holochain sandbox with holoom installed", + "type": "module", + "license": "MIT", + "types": "dist/types/index.d.ts", + "exports": { + ".": { + "types": "./dist/types/index.d.ts", + "node": { + "require": "./dist/index.node.cjs", + "default": "./dist/index.node.js" + } + } + }, + "files": [ + "dist/", + "src/", + "LICENSE", + "README.md" + ], + "repository": "https://github.com/holochain-open-dev/holoom", + "homepage": "https://github.com/holochain-open-dev/holoom/tree/main/packages/sandbox", + "bugs": { + "url": "https://github.com/holochain-open-dev/holoom.git/issues" + }, + "scripts": { + "build": "rimraf dist && npm run build:node", + "build:node": "rollup -c rollup.node.config.ts --configPlugin typescript" + }, + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@rollup/plugin-alias": "^5.1.0", + "@rollup/plugin-json": "^6.1.0", + "@rollup/plugin-typescript": "^11.1.6", + "@types/ws": "^8.5.10", + "rimraf": "^5.0.5", + "rollup": "^4.12.0", + "rollup-plugin-cleanup": "^3.2.1" + }, + "dependencies": { + "@holochain/client": "^0.18.0-dev", + "yaml": "^2.5.1" + } +} diff --git a/packages/sandbox/rollup.node.config.ts b/packages/sandbox/rollup.node.config.ts new file mode 100644 index 0000000..8684cb5 --- /dev/null +++ b/packages/sandbox/rollup.node.config.ts @@ -0,0 +1,48 @@ +import json from "@rollup/plugin-json"; +import typescript from "@rollup/plugin-typescript"; +import * as fs from "fs"; +import cleanup from "rollup-plugin-cleanup"; +import { RollupOptions } from "rollup"; + +const pkg = JSON.parse(fs.readFileSync("./package.json", "utf-8")); +const banner = `/** + * @module ${pkg.name} + * @version ${pkg.version} + * @file ${pkg.description} + * @license ${pkg.license} + * @see [Github]{@link ${pkg.homepage}} +*/`; + +const config: RollupOptions = { + input: "src/index.ts", + output: [ + { + file: pkg.exports["."].node.require, + format: "cjs", + banner, + exports: "auto", + }, + { + file: pkg.exports["."].node.default, + format: "es", + banner, + }, + ], + external: [ + ...Object.keys(pkg.dependencies), + "node:fs/promises", + "node:child_process", + ], + plugins: [ + typescript({ + tsconfig: "./build.tsconfig.json", + }), + cleanup({ comments: "jsdoc" }), + json(), + ], + onwarn: (warning) => { + throw new Error(warning.message); + }, +}; + +export default config; diff --git a/packages/sandbox/src/happ.ts b/packages/sandbox/src/happ.ts new file mode 100644 index 0000000..97e8c57 --- /dev/null +++ b/packages/sandbox/src/happ.ts @@ -0,0 +1,89 @@ +import { + AdminWebsocket, + AppCallZomeRequest, + AppWebsocket, + encodeHashToBase64, + getSigningCredentials, + InstallAppRequest, +} from "@holochain/client"; + +export async function ensureHapp( + adminWs: AdminWebsocket, + happPath: string, + networkSeed: string, + allowedOrigins = "holoom" +): Promise { + const apps = await adminWs.listApps({}); + if (!apps.some((info) => info.installed_app_id === "holoom")) { + // App not installed + await installHapp(adminWs, happPath, networkSeed, allowedOrigins); + } + + const appInterfaces = await adminWs.listAppInterfaces(); + const holoomAppInterface = appInterfaces.find( + (appInterface) => appInterface.installed_app_id === "holoom" + ); + if (!holoomAppInterface) { + throw new Error("Could not find app interface for holoom"); + } + + const issuedToken = await adminWs.issueAppAuthenticationToken({ + installed_app_id: "holoom", + }); + console.log("Issued token"); + + const appWs = await AppWebsocket.connect({ + url: new URL(`http://localhost:${holoomAppInterface.port}`), + wsClientOptions: { origin: "holoom" }, + token: issuedToken.token, + }); + + // set up automatic zome call signing + const callZome = appWs.callZome.bind(appWs); + appWs.callZome = async (req: AppCallZomeRequest, timeout?: number) => { + let cellId; + if ("role_name" in req) { + if (!appWs.cachedAppInfo) { + throw new Error("appWs.cachedAppInfo not set"); + } + cellId = appWs.getCellIdFromRoleName(req.role_name, appWs.cachedAppInfo); + } else { + cellId = req.cell_id; + } + if (!getSigningCredentials(cellId)) { + await adminWs.authorizeSigningCredentials(cellId); + } + return callZome(req, timeout); + }; + + return appWs; +} + +async function installHapp( + adminWs: AdminWebsocket, + happPath: string, + networkSeed: string, + allowedOrigins: string +) { + const agentPubkey = await adminWs.generateAgentPubKey(); + + const installAppRequest: InstallAppRequest = { + path: happPath, + agent_key: agentPubkey, + membrane_proofs: {}, + installed_app_id: "holoom", + network_seed: networkSeed, + }; + console.debug( + `installing holoom for agent ${encodeHashToBase64(agentPubkey)}` + ); + await adminWs.installApp(installAppRequest); + const resp = await adminWs.enableApp({ + installed_app_id: "holoom", + }); + console.log("enableApp", resp); + await adminWs.attachAppInterface({ + allowed_origins: allowedOrigins, + installed_app_id: "holoom", + }); +} diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts new file mode 100644 index 0000000..f8d5861 --- /dev/null +++ b/packages/sandbox/src/index.ts @@ -0,0 +1,44 @@ +import { AdminWebsocket } from "@holochain/client"; +import { + endSandboxProcess, + ensureSandbox, + SandboxOptions, + startSandbox, +} from "./sandbox"; +import { ensureHapp } from "./happ"; +export * from "./happ"; +export * from "./sandbox"; + +export async function ensureAndConnectToHapp( + sandboxPath: string, + happPath: string, + networkSeed: string, + options: SandboxOptions +) { + await ensureSandbox(sandboxPath, options); + + const { process, adminApiUrl } = await startSandbox( + sandboxPath, + options.password + ); + + const adminWs = await AdminWebsocket.connect({ + url: adminApiUrl, + wsClientOptions: { origin: "holoom" }, + }); + console.debug(`connected to Admin API @ ${adminApiUrl.href}\n`); + + const appWs = await ensureHapp(adminWs, happPath, networkSeed); + + const shutdown = async () => { + await adminWs.client.close(); + await appWs.client.close(); + await endSandboxProcess(process); + }; + + return { + adminWs, + appWs, + shutdown, + }; +} diff --git a/packages/sandbox/src/sandbox.ts b/packages/sandbox/src/sandbox.ts new file mode 100644 index 0000000..d4e5125 --- /dev/null +++ b/packages/sandbox/src/sandbox.ts @@ -0,0 +1,167 @@ +import * as fs from "node:fs/promises"; +import { spawn, ChildProcessWithoutNullStreams } from "node:child_process"; +import yaml from "yaml"; + +export interface SandboxOptions { + bootstrapServerUrl: URL; + signalingServerUrl: URL; + password: string; +} + +/** + * Creates a sandbox by invoking `hc sandbox create` at the specified directory + * if the directory doesn't already exist. + * + * @param path The directory of the conductor sandbox + * @param options Configuration for the sandbox + */ +export async function ensureSandbox(path: string, options: SandboxOptions) { + const exists = await fs + .stat(path) + .then(() => true) + .catch(() => false); + if (!exists) { + await createSandbox(path, options); + } +} + +/** + * Creates a sandbox by invoking `hc sandbox create` at the specified + * directory. + * + * @param path The directory of the conductor sandbox + * @param options Configuration for the sandbox + */ +export async function createSandbox(path: string, options: SandboxOptions) { + const pathComps = path.split("/"); + const sandboxDirName = pathComps.pop(); + const parentPath = pathComps.join("/") || "/"; + if (!sandboxDirName) { + throw new Error(`Invalid sandbox path: '${path}'`); + } + const args = [ + "sandbox", + "--piped", + "create", + "--root", + parentPath, + "-d", + sandboxDirName, + "--in-process-lair", + "network", + ]; + if (options?.bootstrapServerUrl) { + args.push("--bootstrap", options.bootstrapServerUrl.href); + } + args.push("webrtc"); + args.push(options.signalingServerUrl.href); + + const createConductorProcess = spawn("hc", args); + createConductorProcess.stdin.write(options.password); + createConductorProcess.stdin.end(); + + const createConductorPromise = new Promise((resolve, reject) => { + createConductorProcess.stdout.on("data", (data: Buffer) => { + console.debug(`creating conductor config\n${data.toString()}`); + const tmpDirMatches = [ + ...data.toString().matchAll(/ConfigRootPath\("(.*?)"\)/g), + ]; + if (tmpDirMatches.length) { + const actualDir = tmpDirMatches[0][1]; + if (path !== actualDir) { + const err = new Error( + `Unexpected sandbox dir '${actualDir}' instead of '${path}'` + ); + console.error(err); + } + } + }); + createConductorProcess.stdout.on("end", () => { + resolve(); + }); + createConductorProcess.stderr.on("data", (err) => { + console.error(`error when creating conductor config: ${err}\n`); + reject(err); + }); + }); + await createConductorPromise; + + // Disable dpki + const conductorConfigPath = `${path}/conductor-config.yaml`; + const conductorConfig = yaml.parse( + await fs.readFile(conductorConfigPath, "utf8") + ); + conductorConfig.dpki.no_dpki = true; + await fs.writeFile(conductorConfigPath, yaml.stringify(conductorConfig)); +} + +/** + * Starts the sandbox's conductor + * + * @param path The path to the sandbox + * @param password The password on the sandbox + * @returns A handle on the conductor process and the conductor's admin API + * websocket url + */ +export async function startSandbox( + path: string, + password: string +): Promise<{ adminApiUrl: URL; process: ChildProcessWithoutNullStreams }> { + const process = spawn("hc", ["sandbox", "--piped", "run", "-e", path]); + process.stdin.write(password); + process.stdin.end(); + + let adminPort = ""; + const startPromise = new Promise((resolve) => { + process.stdout.on("data", (data: Buffer) => { + const conductorLaunched = data + .toString() + .match(/Conductor launched #!\d ({.*})/); + const holochainRunning = data + .toString() + .includes("Connected successfully to a running holochain"); + if (conductorLaunched || holochainRunning) { + if (conductorLaunched) { + const portConfiguration = JSON.parse(conductorLaunched[1]); + adminPort = portConfiguration.admin_port; + console.debug(`starting conductor\n${data}`); + } + if (holochainRunning) { + // this is the last output of the startup process + resolve(); + } + } else { + console.info(data.toString()); + } + }); + + process.stderr.on("data", (data: Buffer) => { + console.info(data.toString()); + }); + }); + await startPromise; + if (!adminPort) { + throw new Error("Admin port not captured"); + } + return { process, adminApiUrl: new URL(`http://localhost:${adminPort}`) }; +} + +/** + * Shuts down the given conductor process. + * @param process The conductor process + */ +export async function endSandboxProcess( + process: ChildProcessWithoutNullStreams +) { + console.debug("shutting down conductor\n"); + const conductorShutDown = new Promise((resolve) => { + process.on("exit", (code) => { + process?.removeAllListeners(); + process?.stdout.removeAllListeners(); + process?.stderr.removeAllListeners(); + resolve(code); + }); + process.kill("SIGINT"); + }); + return conductorShutDown; +} diff --git a/packages/sandbox/tsconfig.json b/packages/sandbox/tsconfig.json new file mode 100644 index 0000000..e87861b --- /dev/null +++ b/packages/sandbox/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src", "rollup.node.config.ts"] +}