Http publisher

.gitignore

@@ -12,4 +12,5 @@ __pycache__
 supervisord.log
 supervisord.pid
 .env
-ops/testnet/.terraform/providers
+ops/testnet/.terraform/providers
+out.txt

CONTRIBUTING.md

@@ -7,6 +7,8 @@ We need the following installed:
  * docker
  * jq
 
+## start stack manually
+
 #### compile contract
 
 The first thing we need to do is compile the smart contract so we have the ABI and can build that into the container:
@@ -83,3 +85,30 @@ NOTE: if you want a fresh installation - then:
 ```bash
 ./stack clean
 ```
+
+## run integration tests locally
+
+In one terminal:
+
+```bash
+./stack reset
+./stack start
+```
+
+In another terminal:
+
+```bash
+export DEBUG=1
+export SKIP_RESET=1
+./stack integration-tests
+```
+
+To run a single test:
+
+```bash
+export DEBUG=1
+export SKIP_RESET=1
+source .env
+cd test
+go test -v -run TestCowsay .
+```

Dockerfile

@@ -11,7 +11,8 @@ RUN apt-get update && \
       libcurl4-openssl-dev \
       jq \
       tzdata \
-      docker.io
+      docker.io \
+      nginx
 RUN curl -sL https://get.bacalhau.org/install.sh | bash
 RUN pip3 install supervisor
 RUN curl -o kubo.tar.gz https://dist.ipfs.tech/kubo/v0.21.0/kubo_v0.21.0_linux-amd64.tar.gz && \
@@ -28,10 +29,12 @@ ENV BACALHAU_API_HOST=localhost
 # nvidia-smi wrapper script which actually runs a container from inside a container
 ADD nvidia-smi /usr/bin/nvidia-smi
 ADD nvidia-container-cli /usr/bin/nvidia-container-cli
+RUN mkdir -p /lilypad-results
 ENTRYPOINT ["/usr/local/bin/modicum"]
 
 FROM modicum AS resource-provider
 ADD ./src/python/supervisord.resourceProvider.conf /etc/supervisord.conf
+ADD ./src/python/nginx.conf /etc/nginx/sites-available/default
 ENTRYPOINT ["/usr/local/bin/supervisord"]
 
 FROM modicum AS mediator

go.mod

@@ -1,8 +1,3 @@
 module github.com/bacalhau-project/lilypad
 
 go 1.20
-
-require (
-	github.com/ichinaski/pxl v0.0.0-20170812084744-4206eb59e8eb // indirect
-	github.com/nsf/termbox-go v1.1.1 // indirect
-)

go.sum

@@ -1,6 +0,0 @@
-github.com/ichinaski/pxl v0.0.0-20170812084744-4206eb59e8eb h1:TXpEkAMms5/LrIxK7/oWpIL7nTnQkTZ/PgTd8+gDRKE=
-github.com/ichinaski/pxl v0.0.0-20170812084744-4206eb59e8eb/go.mod h1:VnaNfBzNu71FX4FTeceuF/fOjvdJqh/cH1daHU8CTNw=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY=
-github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo=

lilypad

@@ -32,6 +32,14 @@ export IMAGE_MODICUM="${DOCKER_REGISTRY}/${IMAGE_MODICUM_NAME}:${VERSION}"
 export IMAGE_RESOURCE_PROVIDER_NAME=${IMAGE_RESOURCE_PROVIDER_NAME:="${IMAGE_BASE}-resource-provider"}
 export IMAGE_RESOURCE_PROVIDER="${DOCKER_REGISTRY}/${IMAGE_RESOURCE_PROVIDER_NAME}:${VERSION}"
 export LILYPAD_NODE_FLAGS=""
+# what URL will the resource provider report to download results from?
+# if left blank - the resource provider will use https://api.ipify.org to know it's public
+# ip address and will use the RESULTS_PORT below
+# if RESULTS_URL is provided then it must line up with the RESULTS_PORT below
+export RESULTS_URL=${RESULTS_URL:=""}
+# what port do we expose the results on?
+# this needs to line up with the 
+export RESULTS_PORT=${RESULTS_PORT:="80"}
 
 if [[ $# -eq 0 ]]; then
   echo "Usage: $0 <command> [args]"
@@ -55,10 +63,13 @@ elif [[ $1 == "serve" ]]; then
   fi
   # /tmp is for bacalhau to be able to find its own results
   docker run -d --restart always --name resource-provider \
+    -p $RESULTS_PORT:80 \
     -v /tmp:/tmp \
     -v /var/run/docker.sock:/var/run/docker.sock \
     $GPU_ARG $GPU_VALUE \
     -e PRIVATE_KEY \
+    -e RESULTS_URL \
+    -e RESULTS_PORT \
     -e CONTRACT_ADDRESS \
     -e CONTRACT_ABI_FILE=/Modicum.json \
     -e MEDIATOR_ADDRESSES \
@@ -68,6 +79,7 @@ elif [[ $1 == "serve" ]]; then
 elif [[ $1 == "run" ]]; then
   shift
   docker run -ti --rm --name submitjob \
+    -e DEBUG \
     -e PRIVATE_KEY \
     -e CONTRACT_ADDRESS \
     -e CONTRACT_ABI_FILE=/Modicum.json \

nvidia-container-cli

@@ -1,6 +1,6 @@
 #!/bin/bash
 docker pull -q nvidia/cuda:11.0.3-devel-ubuntu20.04 >/dev/null
-docker run -i --privileged --gpus all \
+docker run -i --rm --privileged --gpus all \
     -v /usr/bin/nvidia-container-cli:/usr/bin/nvidia-container-cli \
     -v /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1 \
     -v /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.10.0:/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.10.0 \

nvidia-smi

@@ -1,6 +1,6 @@
 #!/bin/bash
 docker pull -q nvidia/cuda:11.0.3-devel-ubuntu20.04 >/dev/null
-docker run -i --gpus all nvidia/cuda:11.0.3-devel-ubuntu20.04 nvidia-smi "$@"
+docker run -i --rm --gpus all nvidia/cuda:11.0.3-devel-ubuntu20.04 nvidia-smi "$@"
 if [ $? -ne 0 ]; then
     echo
     echo "No GPU found"

src/js/contracts/Modicum.sol

@@ -231,6 +231,7 @@ contract Modicum {
     // address[] mediator_index;
 
     mapping(address => ResourceProvider) resourceProviders;
+    mapping(address => string) resourceProviderURLs;
     mapping(address => JobCreator) jobCreators;
 
     ResourceOffer[] resourceOffers;
@@ -411,6 +412,15 @@ contract Modicum {
         emit ResourceProviderAddedSupportedFirstLayer(msg.sender, firstLayerHash);
     }
 
+    function getResourceProviderResultsURL(address id) public view returns (string memory) {
+        return resourceProviderURLs[id];
+    }
+
+    function setResourceProviderResultsURL(address id, string calldata url) public {
+      resourceProviderURLs[id] = url;
+    }
+
+
     // function getResourceProviderTrustedMediators(address rp) public view returns (address[] memory) {
     //     return resourceProviders[rp].trustedMediators;
     // }

src/js/test/onchain-client.js

@@ -209,6 +209,7 @@ describe("Modicum", async () => {
           .registerResourceProvider(
             1, //Architecture arch,
             0, //timePerInstruction
+            'http://1.2.3.4' // resultsURL
           )
         await expect(
           modicumContract
@@ -265,6 +266,7 @@ describe("Modicum", async () => {
           .registerResourceProvider(
             1, //Architecture arch,
             0, //timePerInstruction
+            'http://1.2.3.4' // resultsURL
           )
 
         const postResourceOfferTrx = await modicumContract
@@ -403,6 +405,7 @@ describe("Modicum", async () => {
             .registerResourceProvider(
               1, //Architecture arch,
               0, //timePerInstruction
+              'http://1.2.3.4' // resultsURL
             )
 
           const postResourceOfferTrx = await modicumContract

src/python/modicum/JobCreator.py

@@ -322,6 +322,10 @@ def platformListener(self):
                             self.status = f"❌ {params['hash']}"
                             self.state = "ResultsPosted"
                         else:
+                            # resultsURL = self.ethclient.contract.functions.postJobOfferPartTwo
+                            self.logger.info("🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣🟣 WE ARE HERE 2")
+                            import pprint; pprint.pprint(self.matches[matchID])
+                            import pprint; pprint.pprint(self.job_offers[joid])
                             self.status = f"https://ipfs.io/ipfs/{params['hash']}"
                             self.state = "ResultsPosted"
                         if(should_mediate()):
@@ -448,7 +452,7 @@ def postLilypadOffer(self, template, params):
         self.deposit = deposit
 
         self.status = f"Sending deposit of {Web3.from_wei(self.deposit, 'ether')} lilETH to contract"
-
+        self.logger.info(f"Sending deposit of {Web3.from_wei(self.deposit, 'ether')} lilETH to contract")
         self.logger.info("🔵🔵🔵 post job offer")
         txHash = self.ethclient.transact(
             self.ethclient.contract.functions.postJobOfferPartOne(

src/python/modicum/Mediator.py

@@ -150,6 +150,10 @@ def getJob(self, matchID, JO, execute):
                 ---------------------------------------------------------------------------------------
                 """))
 
+                # before we return the hash - let's copy the results out from IPFS so we can serve them
+                # from out embedded nginx container
+                subprocess.run(['ipfs', '--repo-dir', '/root/.ipfs', 'get', resultHash], text=True, capture_output=True, check=True, cwd="/lilypad-results")
+
                 return resultHash
 
 

src/python/modicum/ResourceProvider.py

@@ -44,11 +44,12 @@ def __init__(self, index=0, sim=False):
         self.scheduler = BackgroundScheduler()
         self.scheduler.start()
 
-    def register(self, account, arch, timePerInstruction):
+    def register(self, account, arch, timePerInstruction, resultsURL):
         self.logger.info("A: registerResourceProvider")
         self.account = account
+        resultsURL = ""
         self.ethclient.transact(
-            self.ethclient.contract.functions.registerResourceProvider(arch, timePerInstruction),
+            self.ethclient.contract.functions.registerResourceProvider(arch, timePerInstruction, resultsURL),
             { "from": self.account },
         )
         return 0

src/python/modicum/cli.py

@@ -5,6 +5,7 @@
 import zmq
 import logging
 import json
+import requests
 from web3 import Web3
 from .JobCreator import JobFinished
 
@@ -484,7 +485,6 @@ def getSize(tag):
     size = DC.getSize(_DIRIP_, _SSHPORT_, username, tag, _SSHKEY_)
     print(size)
 
-
 ################################################################################
 # RP CLI
 ################################################################################
@@ -517,8 +517,16 @@ def startRP(path,index,host,sim,mediator):
     # NOTE: we force index index here so we are only ever using either
     # the first (unlocked) account or the overriden account supplied by the env
     RP.platformConnect(_CONTRACT_ADDRESS_, _GETHIP_, _GETHPORT_, 0)
-    print("Resource Provider Daemon is registering... ")
-    exitcode = RP.register(RP.account,Architecture.amd64.value, 1)# ratio to 1Gz processor # XXX should this be arm64???
+
+    resultsURL = os.environ.get('RESULTS_URL')
+    resultsPort = os.environ.get('RESULTS_PORT')
+    if resultsURL is None:
+        response = requests.get('https://api.ipify.org')
+        publicIP = response.text
+        resultsURL = "http://%s:%s" % (publicIP, resultsPort,)
+
+    print("Resource Provider Daemon is registering with resultsURL: %s" %resultsURL)
+    exitcode = RP.register(RP.account,Architecture.amd64.value, 1, resultsURL)# ratio to 1Gz processor # XXX should this be arm64???
     print("exitcode:  %s" %exitcode)
     while not RP.registered:
         time.sleep(0.1)
@@ -542,6 +550,7 @@ def startRP(path,index,host,sim,mediator):
     while not RP.idle:
         time.sleep(0.1)
 
+    print("Mediator added - now posting resource offer... ")
     exitcode = RP.postDefaultOffer()
 
 @click.command('startRPDaemon')
@@ -689,11 +698,12 @@ def runLilypadCLI(args, template, params, mediator):
     JC = JobCreator.JobCreator(index, False)
 
     # User facing, quiet logging
-    import logging
-    logger = logging.getLogger("JobCreator")
-    logger.setLevel(logging.ERROR)
-    logger = logging.getLogger("EthereumClient")
-    logger.setLevel(logging.ERROR)
+    if os.environ.get('DEBUG') is None:
+        import logging
+        logger = logging.getLogger("JobCreator")
+        logger.setLevel(logging.ERROR)
+        logger = logging.getLogger("EthereumClient")
+        logger.setLevel(logging.ERROR)
 
     print(f"\n🌟 Lilypad submitting job {template}({params}) 🌟\n")
 

src/python/nginx.conf

@@ -0,0 +1,17 @@
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    root /lilypad-results;
+
+    index index.html index.htm index.nginx-debian.html;
+
+    server_name _;
+
+    location / {
+        # First attempt to serve request as file, then
+        # as directory, then fall back to displaying a 404.
+        try_files $uri $uri/ =404;
+    }
+
+}

src/python/supervisord.resourceProvider.conf

@@ -14,6 +14,11 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 
 # our programs
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+autostart=true
+autorestart=true
+
 [program:ipfs]
 command=bash /app/scripts/start-ipfs.sh