diff --git a/.gitignore b/.gitignore index 34590a7..d00757f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ __pycache__ supervisord.log supervisord.pid .env -ops/testnet/.terraform/providers \ No newline at end of file +ops/testnet/.terraform/providers +out.txt \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90cfd6d..d46733b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,8 @@ We need the following installed: * docker * jq +## start stack manually + #### compile contract The first thing we need to do is compile the smart contract so we have the ABI and can build that into the container: @@ -83,3 +85,30 @@ NOTE: if you want a fresh installation - then: ```bash ./stack clean ``` + +## run integration tests locally + +In one terminal: + +```bash +./stack reset +./stack start +``` + +In another terminal: + +```bash +export DEBUG=1 +export SKIP_RESET=1 +./stack integration-tests +``` + +To run a single test: + +```bash +export DEBUG=1 +export SKIP_RESET=1 +source .env +cd test +go test -v -run TestCowsay . +``` \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index dca7237..0c380db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,8 @@ RUN apt-get update && \ libcurl4-openssl-dev \ jq \ tzdata \ - docker.io + docker.io \ + nginx RUN curl -sL https://get.bacalhau.org/install.sh | bash RUN pip3 install supervisor RUN curl -o kubo.tar.gz https://dist.ipfs.tech/kubo/v0.21.0/kubo_v0.21.0_linux-amd64.tar.gz && \ @@ -28,10 +29,12 @@ ENV BACALHAU_API_HOST=localhost # nvidia-smi wrapper script which actually runs a container from inside a container ADD nvidia-smi /usr/bin/nvidia-smi ADD nvidia-container-cli /usr/bin/nvidia-container-cli +RUN mkdir -p /lilypad-results ENTRYPOINT ["/usr/local/bin/modicum"] FROM modicum AS resource-provider ADD ./src/python/supervisord.resourceProvider.conf /etc/supervisord.conf +ADD ./src/python/nginx.conf /etc/nginx/sites-available/default ENTRYPOINT ["/usr/local/bin/supervisord"] FROM modicum AS mediator diff --git a/go.mod b/go.mod index 4e7391e..917480f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,3 @@ module github.com/bacalhau-project/lilypad go 1.20 - -require ( - github.com/ichinaski/pxl v0.0.0-20170812084744-4206eb59e8eb // indirect - github.com/nsf/termbox-go v1.1.1 // indirect -) diff --git a/go.sum b/go.sum index 3e2c66d..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +0,0 @@ -github.com/ichinaski/pxl v0.0.0-20170812084744-4206eb59e8eb h1:TXpEkAMms5/LrIxK7/oWpIL7nTnQkTZ/PgTd8+gDRKE= -github.com/ichinaski/pxl v0.0.0-20170812084744-4206eb59e8eb/go.mod h1:VnaNfBzNu71FX4FTeceuF/fOjvdJqh/cH1daHU8CTNw= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY= -github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo= diff --git a/lilypad b/lilypad index f2bce78..47e53fb 100644 --- a/lilypad +++ b/lilypad @@ -32,6 +32,14 @@ export IMAGE_MODICUM="${DOCKER_REGISTRY}/${IMAGE_MODICUM_NAME}:${VERSION}" export IMAGE_RESOURCE_PROVIDER_NAME=${IMAGE_RESOURCE_PROVIDER_NAME:="${IMAGE_BASE}-resource-provider"} export IMAGE_RESOURCE_PROVIDER="${DOCKER_REGISTRY}/${IMAGE_RESOURCE_PROVIDER_NAME}:${VERSION}" export LILYPAD_NODE_FLAGS="" +# what URL will the resource provider report to download results from? +# if left blank - the resource provider will use https://api.ipify.org to know it's public +# ip address and will use the RESULTS_PORT below +# if RESULTS_URL is provided then it must line up with the RESULTS_PORT below +export RESULTS_URL=${RESULTS_URL:=""} +# what port do we expose the results on? +# this needs to line up with the +export RESULTS_PORT=${RESULTS_PORT:="80"} if [[ $# -eq 0 ]]; then echo "Usage: $0 [args]" @@ -55,10 +63,13 @@ elif [[ $1 == "serve" ]]; then fi # /tmp is for bacalhau to be able to find its own results docker run -d --restart always --name resource-provider \ + -p $RESULTS_PORT:80 \ -v /tmp:/tmp \ -v /var/run/docker.sock:/var/run/docker.sock \ $GPU_ARG $GPU_VALUE \ -e PRIVATE_KEY \ + -e RESULTS_URL \ + -e RESULTS_PORT \ -e CONTRACT_ADDRESS \ -e CONTRACT_ABI_FILE=/Modicum.json \ -e MEDIATOR_ADDRESSES \ @@ -68,6 +79,7 @@ elif [[ $1 == "serve" ]]; then elif [[ $1 == "run" ]]; then shift docker run -ti --rm --name submitjob \ + -e DEBUG \ -e PRIVATE_KEY \ -e CONTRACT_ADDRESS \ -e CONTRACT_ABI_FILE=/Modicum.json \ diff --git a/nvidia-container-cli b/nvidia-container-cli index c11cd7c..084eab8 100755 --- a/nvidia-container-cli +++ b/nvidia-container-cli @@ -1,6 +1,6 @@ #!/bin/bash docker pull -q nvidia/cuda:11.0.3-devel-ubuntu20.04 >/dev/null -docker run -i --privileged --gpus all \ +docker run -i --rm --privileged --gpus all \ -v /usr/bin/nvidia-container-cli:/usr/bin/nvidia-container-cli \ -v /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1 \ -v /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.10.0:/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.10.0 \ diff --git a/nvidia-smi b/nvidia-smi index bce93fc..d8d272f 100755 --- a/nvidia-smi +++ b/nvidia-smi @@ -1,6 +1,6 @@ #!/bin/bash docker pull -q nvidia/cuda:11.0.3-devel-ubuntu20.04 >/dev/null -docker run -i --gpus all nvidia/cuda:11.0.3-devel-ubuntu20.04 nvidia-smi "$@" +docker run -i --rm --gpus all nvidia/cuda:11.0.3-devel-ubuntu20.04 nvidia-smi "$@" if [ $? -ne 0 ]; then echo echo "No GPU found" diff --git a/src/js/contracts/Modicum.sol b/src/js/contracts/Modicum.sol index 1c579c3..94a9034 100644 --- a/src/js/contracts/Modicum.sol +++ b/src/js/contracts/Modicum.sol @@ -231,6 +231,7 @@ contract Modicum { // address[] mediator_index; mapping(address => ResourceProvider) resourceProviders; + mapping(address => string) resourceProviderURLs; mapping(address => JobCreator) jobCreators; ResourceOffer[] resourceOffers; @@ -411,6 +412,15 @@ contract Modicum { emit ResourceProviderAddedSupportedFirstLayer(msg.sender, firstLayerHash); } + function getResourceProviderResultsURL(address id) public view returns (string memory) { + return resourceProviderURLs[id]; + } + + function setResourceProviderResultsURL(address id, string calldata url) public { + resourceProviderURLs[id] = url; + } + + // function getResourceProviderTrustedMediators(address rp) public view returns (address[] memory) { // return resourceProviders[rp].trustedMediators; // } diff --git a/src/js/test/onchain-client.js b/src/js/test/onchain-client.js index 263f47b..66b7a36 100644 --- a/src/js/test/onchain-client.js +++ b/src/js/test/onchain-client.js @@ -209,6 +209,7 @@ describe("Modicum", async () => { .registerResourceProvider( 1, //Architecture arch, 0, //timePerInstruction + 'http://1.2.3.4' // resultsURL ) await expect( modicumContract @@ -265,6 +266,7 @@ describe("Modicum", async () => { .registerResourceProvider( 1, //Architecture arch, 0, //timePerInstruction + 'http://1.2.3.4' // resultsURL ) const postResourceOfferTrx = await modicumContract @@ -403,6 +405,7 @@ describe("Modicum", async () => { .registerResourceProvider( 1, //Architecture arch, 0, //timePerInstruction + 'http://1.2.3.4' // resultsURL ) const postResourceOfferTrx = await modicumContract diff --git a/src/python/modicum/JobCreator.py b/src/python/modicum/JobCreator.py index 1c40e8b..620ee0f 100644 --- a/src/python/modicum/JobCreator.py +++ b/src/python/modicum/JobCreator.py @@ -322,6 +322,10 @@ def platformListener(self): self.status = f"āŒ {params['hash']}" self.state = "ResultsPosted" else: + # resultsURL = self.ethclient.contract.functions.postJobOfferPartTwo + self.logger.info("šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£šŸŸ£ WE ARE HERE 2") + import pprint; pprint.pprint(self.matches[matchID]) + import pprint; pprint.pprint(self.job_offers[joid]) self.status = f"https://ipfs.io/ipfs/{params['hash']}" self.state = "ResultsPosted" if(should_mediate()): @@ -448,7 +452,7 @@ def postLilypadOffer(self, template, params): self.deposit = deposit self.status = f"Sending deposit of {Web3.from_wei(self.deposit, 'ether')} lilETH to contract" - + self.logger.info(f"Sending deposit of {Web3.from_wei(self.deposit, 'ether')} lilETH to contract") self.logger.info("šŸ”µšŸ”µšŸ”µ post job offer") txHash = self.ethclient.transact( self.ethclient.contract.functions.postJobOfferPartOne( diff --git a/src/python/modicum/Mediator.py b/src/python/modicum/Mediator.py index f0a0387..4834487 100644 --- a/src/python/modicum/Mediator.py +++ b/src/python/modicum/Mediator.py @@ -150,6 +150,10 @@ def getJob(self, matchID, JO, execute): --------------------------------------------------------------------------------------- """)) + # before we return the hash - let's copy the results out from IPFS so we can serve them + # from out embedded nginx container + subprocess.run(['ipfs', '--repo-dir', '/root/.ipfs', 'get', resultHash], text=True, capture_output=True, check=True, cwd="/lilypad-results") + return resultHash diff --git a/src/python/modicum/ResourceProvider.py b/src/python/modicum/ResourceProvider.py index 5b184ca..4227165 100644 --- a/src/python/modicum/ResourceProvider.py +++ b/src/python/modicum/ResourceProvider.py @@ -44,11 +44,12 @@ def __init__(self, index=0, sim=False): self.scheduler = BackgroundScheduler() self.scheduler.start() - def register(self, account, arch, timePerInstruction): + def register(self, account, arch, timePerInstruction, resultsURL): self.logger.info("A: registerResourceProvider") self.account = account + resultsURL = "" self.ethclient.transact( - self.ethclient.contract.functions.registerResourceProvider(arch, timePerInstruction), + self.ethclient.contract.functions.registerResourceProvider(arch, timePerInstruction, resultsURL), { "from": self.account }, ) return 0 diff --git a/src/python/modicum/cli.py b/src/python/modicum/cli.py index 0a83ead..d9e2761 100644 --- a/src/python/modicum/cli.py +++ b/src/python/modicum/cli.py @@ -5,6 +5,7 @@ import zmq import logging import json +import requests from web3 import Web3 from .JobCreator import JobFinished @@ -484,7 +485,6 @@ def getSize(tag): size = DC.getSize(_DIRIP_, _SSHPORT_, username, tag, _SSHKEY_) print(size) - ################################################################################ # RP CLI ################################################################################ @@ -517,8 +517,16 @@ def startRP(path,index,host,sim,mediator): # NOTE: we force index index here so we are only ever using either # the first (unlocked) account or the overriden account supplied by the env RP.platformConnect(_CONTRACT_ADDRESS_, _GETHIP_, _GETHPORT_, 0) - print("Resource Provider Daemon is registering... ") - exitcode = RP.register(RP.account,Architecture.amd64.value, 1)# ratio to 1Gz processor # XXX should this be arm64??? + + resultsURL = os.environ.get('RESULTS_URL') + resultsPort = os.environ.get('RESULTS_PORT') + if resultsURL is None: + response = requests.get('https://api.ipify.org') + publicIP = response.text + resultsURL = "http://%s:%s" % (publicIP, resultsPort,) + + print("Resource Provider Daemon is registering with resultsURL: %s" %resultsURL) + exitcode = RP.register(RP.account,Architecture.amd64.value, 1, resultsURL)# ratio to 1Gz processor # XXX should this be arm64??? print("exitcode: %s" %exitcode) while not RP.registered: time.sleep(0.1) @@ -542,6 +550,7 @@ def startRP(path,index,host,sim,mediator): while not RP.idle: time.sleep(0.1) + print("Mediator added - now posting resource offer... ") exitcode = RP.postDefaultOffer() @click.command('startRPDaemon') @@ -689,11 +698,12 @@ def runLilypadCLI(args, template, params, mediator): JC = JobCreator.JobCreator(index, False) # User facing, quiet logging - import logging - logger = logging.getLogger("JobCreator") - logger.setLevel(logging.ERROR) - logger = logging.getLogger("EthereumClient") - logger.setLevel(logging.ERROR) + if os.environ.get('DEBUG') is None: + import logging + logger = logging.getLogger("JobCreator") + logger.setLevel(logging.ERROR) + logger = logging.getLogger("EthereumClient") + logger.setLevel(logging.ERROR) print(f"\nšŸŒŸ Lilypad submitting job {template}({params}) šŸŒŸ\n") diff --git a/src/python/nginx.conf b/src/python/nginx.conf new file mode 100644 index 0000000..98d89e2 --- /dev/null +++ b/src/python/nginx.conf @@ -0,0 +1,17 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + + root /lilypad-results; + + index index.html index.htm index.nginx-debian.html; + + server_name _; + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + } + +} diff --git a/src/python/supervisord.resourceProvider.conf b/src/python/supervisord.resourceProvider.conf index 53b9a65..f459410 100644 --- a/src/python/supervisord.resourceProvider.conf +++ b/src/python/supervisord.resourceProvider.conf @@ -14,6 +14,11 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface # our programs +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +autostart=true +autorestart=true + [program:ipfs] command=bash /app/scripts/start-ipfs.sh diff --git a/stack b/stack index 9ffecec..686bf6f 100755 --- a/stack +++ b/stack @@ -20,6 +20,15 @@ export IMAGE_HARDHAT=${IMAGE_HARDHAT:="${IMAGE_BASE}-hardhat"} export DEPLOYMENT=${DEPLOYMENT:="localgeth"} export RPC_URL=${RPC_URL:="http://geth:8545"} +# what URL will the resource provider report to download results from? +# if left blank - the resource provider will use https://api.ipify.org to know it's public +# ip address and will use the RESULTS_PORT below +# if RESULTS_URL is provided then it must line up with the RESULTS_PORT below +export RESULTS_URL=${RESULTS_URL:=""} +# what port do we expose the results on? +# this needs to line up with the +export RESULTS_PORT=${RESULTS_PORT:="8080"} + export VERSION=${VERSION:="$(cd $DIR; git rev-parse --short HEAD)-$(uname -m |sed 's/arm64/aarch64/')"} export IMAGE_MODICUM_NAME=${IMAGE_MODICUM_NAME:="${IMAGE_BASE}-modicum"} @@ -43,6 +52,16 @@ function gSCP() { gcloud compute scp --quiet --zone=$GCP_ZONE $1 $GCP_NODE_NAME:$2 } +function install() { + ( + set -euo pipefail + cd src/js + if [[ ! -d node_modules ]]; then + npm install + fi + ) +} + function upload-stack() { gSCP ./stack ./stack } @@ -90,10 +109,10 @@ function contract-address() { function compile-contract() { source ${DIR}/.env + install ( set -euo pipefail cd src/js - npm install npx hardhat compile ) } @@ -143,6 +162,7 @@ function geth() { ethereum/client-go \ --datadir /data/geth \ --dev \ + --miner.gasprice 1 \ --http \ --http.api web3,eth,net \ --http.addr 0.0.0.0 \ @@ -262,10 +282,11 @@ function lilypad-node() { # this is for dev trim if [[ "$DATA_DIRECTORY" == "$DEFAULT_DATA_DIR" ]]; then LILYPAD_NODE_FLAGS="$LILYPAD_NODE_FLAGS -v $DIR/src/python:/app" - - # Workaround https://github.com/bacalhau-project/lilypad/issues/39 - docker run -i --entrypoint /bin/bash -v $DIR/src/python:/app \ + if [[ ! -d "$DIR/src/python/Modicum.egg-info" ]]; then + # Workaround https://github.com/bacalhau-project/lilypad/issues/39 + docker run -i --rm --entrypoint /bin/bash -v $DIR/src/python:/app \ $LILYPAD_NODE_IMAGE -c "pip install -e ." + fi else echo >&2 "running in production mode..." fi @@ -274,6 +295,19 @@ function lilypad-node() { elif [[ "$CONTAINER_MODE" == "cli" ]]; then LILYPAD_NODE_FLAGS="$LILYPAD_NODE_FLAGS -i --rm" fi + if [[ "$CONTAINER_MODE" == "server" ]]; then + LILYPAD_NODE_FLAGS="$LILYPAD_NODE_FLAGS -d --restart always" + elif [[ "$CONTAINER_MODE" == "cli" ]]; then + LILYPAD_NODE_FLAGS="$LILYPAD_NODE_FLAGS -i --rm" + fi + # expose port 5050 for the results nginx + if [[ "$LILYPAD_NODE_NAME" == "resource-provider" ]]; then + LILYPAD_NODE_FLAGS="$LILYPAD_NODE_FLAGS -p $RESULTS_PORT:80" + # this is for dev trim + if [[ "$DATA_DIRECTORY" == "$DEFAULT_DATA_DIR" ]]; then + export RESULTS_URL="http://localhost:$RESULTS_PORT" + fi + fi GPU_ARG="" GPU_VALUE="" if [ -f "/usr/bin/nvidia-smi" ]; then @@ -287,6 +321,8 @@ function lilypad-node() { -e DEBUG \ -e PRIVATE_KEY \ -e BAD_ACTOR \ + -e RESULTS_URL \ + -e RESULTS_PORT \ -e MEDIATION_CHANCE_PERCENT \ -e MEDIATOR_ADDRESSES \ -e CONTRACT_ADDRESS=$(contract-address) \ @@ -357,11 +393,7 @@ function reset() { set -x docker rm -f faucet-proxy faucet resource-provider mediator solver geth-proxy geth ||true sudo rm -rf $DATA_DIRECTORY - local sleepSeconds="1" - if [[ -z "$QUICK_RESET" ]]; then - (cd src/js && npm install) - sleepSeconds="5" - fi + install node src/js/scripts/create-new-accounts.js > .env source ${DIR}/.env rm -rf src/js/deployments/localgeth @@ -373,7 +405,7 @@ function reset() { build geth geth-proxy - sleep $sleepSeconds + sleep 1 fund-admin fund-faucet fund-services diff --git a/test/integration_test.go b/test/integration_test.go index 1f1529c..6d1a55b 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -106,7 +106,6 @@ func TestSDXLColours(t *testing.T) { } // TODO: LLM test - func testJob(t *testing.T, args []string, kind string, expectedText string, relPath string) string { maybeReset(t) @@ -132,12 +131,15 @@ func testJob(t *testing.T, args []string, kind string, expectedText string, relP } } }() - out, err := exec.Command("./stack", args...).CombinedOutput() + stdout, stderr, err := runCommandWithOutput("./stack", args) stop <- struct{}{} + if stderr != "" { + log.Printf("stderr: %s", stderr) + } if err != nil { - t.Fatal(err, string(out)) + t.Fatal(err, stdout) } - writeOutput(out, "out.txt") + writeOutput([]byte(stdout), "out.txt") log.Println("----> FINDING IPFS URL") ipfsURL, err := findIPFSURL("out.txt") @@ -192,53 +194,3 @@ func testJob(t *testing.T, args []string, kind string, expectedText string, relP } return cid } - -func runCommand(name string, args []string) error { - log.Printf("Running %s %s", name, args) - cmd := exec.Command(name, args...) - cmd.Dir = PATH - o, err := cmd.CombinedOutput() - if err != nil { - log.Printf("Error from cmd %s %s: %s %s", name, args, err, o) - } - return err -} - -func writeOutput(out []byte, filename string) { - file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - fmt.Println(err) - return - } - defer file.Close() - if _, err := file.Write(out); err != nil { - fmt.Println(err) - } -} - -func findIPFSURL(filename string) (string, error) { - file, err := os.Open(filename) - if err != nil { - return "", err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) - - for scanner.Scan() { - line := scanner.Text() - log.Print(line) - if strings.Contains(line, "ipfs.io") { - xs := strings.Split(line, " ") - return xs[len(xs)-1], nil - } - } - - if err := scanner.Err(); err != nil { - return "", err - } - - return "", fmt.Errorf("no IPFS URL found in %s", filename) -} diff --git a/test/utils.go b/test/utils.go new file mode 100644 index 0000000..31c676c --- /dev/null +++ b/test/utils.go @@ -0,0 +1,92 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "strings" +) + +type MultiWriter struct { + writers []io.Writer +} + +func (mw *MultiWriter) Write(p []byte) (n int, err error) { + for _, w := range mw.writers { + if n, err = w.Write(p); err != nil { + return + } + if n != len(p) { + err = io.ErrShortWrite + return + } + } + return len(p), nil +} + +func NewMultiWriter(writers ...io.Writer) *MultiWriter { + return &MultiWriter{writers: writers} +} + +func runCommand(name string, args []string) error { + log.Printf("Running %s %s", name, args) + _, _, err := runCommandWithOutput(name, args) + return err +} + +func writeOutput(out []byte, filename string) { + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Println(err) + return + } + defer file.Close() + if _, err := file.Write(out); err != nil { + fmt.Println(err) + } +} + +func findIPFSURL(filename string) (string, error) { + file, err := os.Open(filename) + if err != nil { + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + + for scanner.Scan() { + line := scanner.Text() + log.Print(line) + if strings.Contains(line, "ipfs.io") { + xs := strings.Split(line, " ") + return xs[len(xs)-1], nil + } + } + + if err := scanner.Err(); err != nil { + return "", err + } + + return "", fmt.Errorf("no IPFS URL found in %s", filename) +} + +func runCommandWithOutput(name string, args []string) (string, string, error) { + var stdoutBuf, stderrBuf bytes.Buffer + cmd := exec.Command(name, args...) + + cmd.Stdout = NewMultiWriter(os.Stdout, &stdoutBuf) + cmd.Stderr = NewMultiWriter(os.Stderr, &stderrBuf) + + if err := cmd.Run(); err != nil { + return "", "", err + } + + return stdoutBuf.String(), stderrBuf.String(), nil +}