From 24ecb2156aa71137c04e45d344eaf1b5dadcb3b8 Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sat, 27 Feb 2021 17:04:00 -0300 Subject: [PATCH 1/7] user leasehold isn't allowing psql commands --- .gitignore | 2 +- Dockerfile | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/.gitignore b/.gitignore index 9240b19..877c800 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ node_modules/ logs/ - +*.gz diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b7b98d0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:20.04 + +# not to prompt `Configuring tzdata` +ARG DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /home/leasehold/leasehold-core + +# Necessary requirements +RUN apt-get -qq update > /dev/null +RUN apt install -qqy git curl wget build-essential gzip lsb-release > /dev/null +RUN useradd -ms /bin/bash leasehold + +RUN wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/PostgreSQL.list' + +RUN apt-get -qq update > /dev/null +RUN apt install -qqy postgresql-10 nodejs > /dev/null + +RUN sed -i 's/max_connections = 100/max_connections = 300/g' /etc/postgresql/10/main/postgresql.conf +RUN sed -i 's/shared_buffers = 128MB/shared_buffers = 256MB/g' /etc/postgresql/10/main/postgresql.conf + +COPY . /home/leasehold/leasehold-core/ +WORKDIR /home/leasehold/leasehold-core/ + +USER postgres + +RUN service postgresql start && \ + psql -c "CREATE USER leasehold" && \ + psql -c "ALTER ROLE leasehold WITH SUPERUSER" && \ + psql -c "CREATE DATABASE lisk_main OWNER leasehold" && \ + psql -c "CREATE DATABASE leasehold_main OWNER leasehold" + psql -c "ALTER USER leasehold WITH PASSWORD 'password'" + # createdb lisk_main --owner leasehold && \ + # createdb leasehold_main --owner leasehold + +USER leasehold + +RUN service postgresql start && \ + gzip --decompress --to-stdout ./lisk_main_backup-14576795.gz | psql lisk_main -U postgres -w && \ + gzip --decompress --to-stdout ./leasehold_main_backup_27022021.gz | psql leasehold_main -U postgres -w && \ + + +# # DB Snapshots +# RUN rm -f ./lisk_main_backup-14576795.gz +# RUN rm -f ./leasehold_main_backup_27022021.gz + + +# # # NPM +# RUN npm install +# RUN npm install pm2 -g +# RUN npm i -g lisk-commander@2.2.3 +# RUN lisk config:set api.nodes http://2.56.213.101:8010/ + +CMD ["pm2", "start", "leasehold-core-mainnet"] From 12da2c1dd8f2bca51071c1cc3aae325692fc9d79 Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sat, 27 Feb 2021 18:55:19 -0300 Subject: [PATCH 2/7] I think I'm getting closer --- Dockerfile | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index b7b98d0..83f4c95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,22 +3,17 @@ FROM ubuntu:20.04 # not to prompt `Configuring tzdata` ARG DEBIAN_FRONTEND=noninteractive -RUN mkdir -p /home/leasehold/leasehold-core - -# Necessary requirements -RUN apt-get -qq update > /dev/null -RUN apt install -qqy git curl wget build-essential gzip lsb-release > /dev/null -RUN useradd -ms /bin/bash leasehold - -RUN wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/PostgreSQL.list' - -RUN apt-get -qq update > /dev/null -RUN apt install -qqy postgresql-10 nodejs > /dev/null - -RUN sed -i 's/max_connections = 100/max_connections = 300/g' /etc/postgresql/10/main/postgresql.conf -RUN sed -i 's/shared_buffers = 128MB/shared_buffers = 256MB/g' /etc/postgresql/10/main/postgresql.conf - -COPY . /home/leasehold/leasehold-core/ +RUN mkdir -p /home/leasehold/leasehold-core; \ + useradd -ms /bin/bash leasehold; \ + apt-get update; \ + apt-get install -qqy git curl wget build-essential gzip lsb-release; \ + wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg ain" > /etc/apt/sources.list.d/PostgreSQL.list'; \ + apt-get update; \ + apt-get install -qqy postgresql-10 nodejs; \ + sed -i 's/max_connections = 100/max_connections = 300/g' /etc/postgresql/10/main/postgresql.conf; \ + sed -i 's/shared_buffers = 128MB/shared_buffers = 256MB/g' /etc/postgresql/10/main/postgresql.conf + +# COPY . /home/leasehold/leasehold-core/ WORKDIR /home/leasehold/leasehold-core/ USER postgres @@ -27,16 +22,16 @@ RUN service postgresql start && \ psql -c "CREATE USER leasehold" && \ psql -c "ALTER ROLE leasehold WITH SUPERUSER" && \ psql -c "CREATE DATABASE lisk_main OWNER leasehold" && \ - psql -c "CREATE DATABASE leasehold_main OWNER leasehold" + psql -c "CREATE DATABASE leasehold_main OWNER leasehold" && \ psql -c "ALTER USER leasehold WITH PASSWORD 'password'" # createdb lisk_main --owner leasehold && \ # createdb leasehold_main --owner leasehold USER leasehold -RUN service postgresql start && \ - gzip --decompress --to-stdout ./lisk_main_backup-14576795.gz | psql lisk_main -U postgres -w && \ - gzip --decompress --to-stdout ./leasehold_main_backup_27022021.gz | psql leasehold_main -U postgres -w && \ +# RUN service postgresql start && \ +# RUN gzip --decompress --to-stdout ./lisk_main_backup-14576795.gz | psql lisk_main -w && \ +# gzip --decompress --to-stdout ./leasehold_main_backup_27022021.gz | psql leasehold_main -w && \ # # DB Snapshots From 28d8ada7e368101cfa542c87568cfe00c9cdd30c Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sat, 27 Feb 2021 19:04:26 -0300 Subject: [PATCH 3/7] this works so far --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 83f4c95..c078b8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN mkdir -p /home/leasehold/leasehold-core; \ useradd -ms /bin/bash leasehold; \ apt-get update; \ apt-get install -qqy git curl wget build-essential gzip lsb-release; \ - wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg ain" > /etc/apt/sources.list.d/PostgreSQL.list'; \ + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/PostgreSQL.list'; \ apt-get update; \ apt-get install -qqy postgresql-10 nodejs; \ sed -i 's/max_connections = 100/max_connections = 300/g' /etc/postgresql/10/main/postgresql.conf; \ From 7d625b6f9aea79be15fe72ac69fd247424f1eafe Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sun, 28 Feb 2021 09:29:01 -0300 Subject: [PATCH 4/7] seems to be good --- .dockerignore | 1 + Dockerfile | 31 ++++++++++--------------------- 2 files changed, 11 insertions(+), 21 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..10d00b5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +*.gz diff --git a/Dockerfile b/Dockerfile index c078b8d..ac5036c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,18 +3,17 @@ FROM ubuntu:20.04 # not to prompt `Configuring tzdata` ARG DEBIAN_FRONTEND=noninteractive -RUN mkdir -p /home/leasehold/leasehold-core; \ - useradd -ms /bin/bash leasehold; \ +RUN useradd -ms /bin/bash leasehold; \ apt-get update; \ apt-get install -qqy git curl wget build-essential gzip lsb-release; \ + curl -sL https://deb.nodesource.com/setup_10.x | bash -; \ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/PostgreSQL.list'; \ apt-get update; \ apt-get install -qqy postgresql-10 nodejs; \ sed -i 's/max_connections = 100/max_connections = 300/g' /etc/postgresql/10/main/postgresql.conf; \ - sed -i 's/shared_buffers = 128MB/shared_buffers = 256MB/g' /etc/postgresql/10/main/postgresql.conf - -# COPY . /home/leasehold/leasehold-core/ -WORKDIR /home/leasehold/leasehold-core/ + sed -i 's/shared_buffers = 128MB/shared_buffers = 256MB/g' /etc/postgresql/10/main/postgresql.conf; \ + npm install pm2 -g; \ + npm i -g lisk-commander@2.2.3 USER postgres @@ -24,25 +23,15 @@ RUN service postgresql start && \ psql -c "CREATE DATABASE lisk_main OWNER leasehold" && \ psql -c "CREATE DATABASE leasehold_main OWNER leasehold" && \ psql -c "ALTER USER leasehold WITH PASSWORD 'password'" - # createdb lisk_main --owner leasehold && \ - # createdb leasehold_main --owner leasehold USER leasehold -# RUN service postgresql start && \ -# RUN gzip --decompress --to-stdout ./lisk_main_backup-14576795.gz | psql lisk_main -w && \ -# gzip --decompress --to-stdout ./leasehold_main_backup_27022021.gz | psql leasehold_main -w && \ - - -# # DB Snapshots -# RUN rm -f ./lisk_main_backup-14576795.gz -# RUN rm -f ./leasehold_main_backup_27022021.gz +RUN mkdir -p /home/leasehold/leasehold-core +COPY . /home/leasehold/leasehold-core/ +WORKDIR /home/leasehold/leasehold-core/ -# # # NPM -# RUN npm install -# RUN npm install pm2 -g -# RUN npm i -g lisk-commander@2.2.3 -# RUN lisk config:set api.nodes http://2.56.213.101:8010/ +RUN npm install; \ + lisk config:set api.nodes http://2.56.213.101:8010/ CMD ["pm2", "start", "leasehold-core-mainnet"] From 7dd5c266954931dc2b483d8d02de8b42a996623a Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sun, 28 Feb 2021 19:30:47 -0300 Subject: [PATCH 5/7] it's working --- .dockerignore | 1 - Dockerfile | 29 +++++++++++--- start.js | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++ start.sh | 4 ++ 4 files changed, 133 insertions(+), 7 deletions(-) delete mode 100644 .dockerignore create mode 100644 start.js create mode 100644 start.sh diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 10d00b5..0000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -*.gz diff --git a/Dockerfile b/Dockerfile index ac5036c..46146c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,15 +5,17 @@ ARG DEBIAN_FRONTEND=noninteractive RUN useradd -ms /bin/bash leasehold; \ apt-get update; \ - apt-get install -qqy git curl wget build-essential gzip lsb-release; \ + apt-get install -qqy git curl wget build-essential gzip lsb-release sudo; \ curl -sL https://deb.nodesource.com/setup_10.x | bash -; \ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main" > /etc/apt/sources.list.d/PostgreSQL.list'; \ apt-get update; \ apt-get install -qqy postgresql-10 nodejs; \ sed -i 's/max_connections = 100/max_connections = 300/g' /etc/postgresql/10/main/postgresql.conf; \ sed -i 's/shared_buffers = 128MB/shared_buffers = 256MB/g' /etc/postgresql/10/main/postgresql.conf; \ + echo 'leasehold ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers; \ npm install pm2 -g; \ - npm i -g lisk-commander@2.2.3 + npm i -g lisk-commander@2.2.3; \ + usermod -aG sudo leasehold USER postgres @@ -27,11 +29,26 @@ RUN service postgresql start && \ USER leasehold RUN mkdir -p /home/leasehold/leasehold-core +RUN chown -R leasehold:leasehold /home/leasehold -COPY . /home/leasehold/leasehold-core/ -WORKDIR /home/leasehold/leasehold-core/ +COPY --chown=leasehold . /home/leasehold/leasehold-core/ +WORKDIR /home/leasehold/ -RUN npm install; \ +RUN sudo service postgresql start && \ + gzip --decompress --to-stdout ./leasehold-core/leasehold_main_backup_27022021.gz | psql -U leasehold leasehold_main -w && \ + gzip --decompress --to-stdout ./leasehold-core/lisk_main_backup-14576795 | psql -U leasehold lisk_main -w -w; \ + rm -f ./leasehold-core/leasehold_main_backup_27022021.gz; \ + rm -f ./leasehold-core/lisk_main_backup-14576795 + +RUN npm install --prefix ./leasehold-core; \ lisk config:set api.nodes http://2.56.213.101:8010/ -CMD ["pm2", "start", "leasehold-core-mainnet"] +EXPOSE 5432 +EXPOSE 8001 +EXPOSE 8010 + +RUN ls + +RUN chmod +x ./leasehold-core/start.sh + +CMD ["./leasehold-core/start.sh"] diff --git a/start.js b/start.js new file mode 100644 index 0000000..103b83f --- /dev/null +++ b/start.js @@ -0,0 +1,106 @@ +const util = require('util'); +const { spawn } = require('child_process'); +const readline = require('readline'); +const exec = util.promisify(require('child_process').exec); + +const NETWORK = 'mainnet'; +const LEASEHOLD_SNAPSHOT = 'leasehold_main_backup_27022021.gz'; +const LISK_SNAPSHOT = 'lisk_main_backup-14576795.gz'; +const CONTAINER_NAME = 'leasehold-core'; +const IMAGE_NAME = 'leasehold'; + +const commandLog = (msg, warning = false) => + console.log(`\n${warning ? '\x1b[31m' : ''}\x1b[1m${msg}\x1b[0m`); + +const errorLog = (msg) => console.error(`\x1b[31m\x1b[1m${msg}\x1b[0m`); + +const execCommand = (command) => { + console.log(`Executing command: ${command}`); + return new Promise((res, rej) => { + command = command.split(' '); + const process = spawn(command[0], command.slice(1)); + readline + .createInterface({ + input: process.stdout, + terminal: false, + }) + .on('line', function (line) { + console.log(line); + }); + process.on('close', (code) => res(code)); + process.on('error', (err) => rej(err)); + }); + + // const { stdout, stderr } = await exec(command, { shell: true }); + // if (stderr) { + // errorLog(stderr); + // } + + // console.log(stdout); +}; + +(async () => { + try { + const { stdout: ls } = await exec(`ls`); + if (!ls.includes(LISK_SNAPSHOT)) { + commandLog( + 'IMPORTANT: Getting Lisk snapshot. This process might take a while!', + true, + ); + await execCommand( + `wget --no-check-certificate https://snapshots.lisk.io/${NETWORK}/${LISK_SNAPSHOT}`, + ); + } else { + commandLog(`Skipping ${LISK_SNAPSHOT} it already exists.`); + } + + if (!ls.includes(LEASEHOLD_SNAPSHOT)) { + commandLog( + 'IMPORTANT: Getting Leasehold snapshot. This process might take a while!', + true, + ); + await execCommand( + `wget --no-check-certificate https://testnet.leasehold.io/snapshots/${NETWORK}/${LEASEHOLD_SNAPSHOT}`, + ); + } else { + commandLog(`Skipping ${LEASEHOLD_SNAPSHOT} it already exists.`); + } + + commandLog('Building docker container'); + await execCommand(`docker build -t ${IMAGE_NAME} .`); + + commandLog('Removing container if exists.'); + await execCommand(`docker rm --force ${CONTAINER_NAME}`); + + commandLog(`Running container with name ${CONTAINER_NAME}`); + await execCommand( + `docker run -d -p 8010:8010 -p 8001:8001 -p 5440:5432 --name ${CONTAINER_NAME} ${IMAGE_NAME}`, + ); + + commandLog('Getting IP of container'); + const { stdout } = await exec(`docker inspect ${CONTAINER_NAME}`); + const ip = JSON.parse(stdout)[0].NetworkSettings.IPAddress; + if (ip === '') + throw new Error('No IP Address available, container probably exited.'); + + commandLog(`IP Found: ${ip}`) + + // commandLog( + // 'IMPORTANT: Writing the snapshot to the DB. This process might take a while!', + // true, + // ); + // await execCommand( + // `gzip --decompress --to-stdout ./${LISK_SNAPSHOT} | psql -U leasehold -h ${ip} lisk_main -w`, + // ); + + // commandLog( + // 'IMPORTANT: Writing the snapshot to the DB. This process might take a while!', + // true, + // ); + // await execCommand( + // `gzip --decompress --to-stdout ./${LEASEHOLD_SNAPSHOT} | psql -U leasehold -h ${ip} leasehold_main -w`, + // ); + } catch (err) { + errorLog(err.message); + } +})(); diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..592a7ae --- /dev/null +++ b/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +sudo /usr/sbin/service postgresql start +/usr/bin/pm2-runtime start leasehold-core From a66f966ef0a610173aa673bd67058efc48400af5 Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sun, 28 Feb 2021 19:35:13 -0300 Subject: [PATCH 6/7] might to will and delete postgres port, it irrelevant --- start.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/start.js b/start.js index 103b83f..f73d2e2 100644 --- a/start.js +++ b/start.js @@ -44,7 +44,7 @@ const execCommand = (command) => { const { stdout: ls } = await exec(`ls`); if (!ls.includes(LISK_SNAPSHOT)) { commandLog( - 'IMPORTANT: Getting Lisk snapshot. This process might take a while!', + 'IMPORTANT: Getting Lisk snapshot. This process will take a while!', true, ); await execCommand( @@ -56,7 +56,7 @@ const execCommand = (command) => { if (!ls.includes(LEASEHOLD_SNAPSHOT)) { commandLog( - 'IMPORTANT: Getting Leasehold snapshot. This process might take a while!', + 'IMPORTANT: Getting Leasehold snapshot. This process will take a while!', true, ); await execCommand( @@ -67,6 +67,10 @@ const execCommand = (command) => { } commandLog('Building docker container'); + commandLog( + 'IMPORTANT: This process will take a while!', + true, + ); await execCommand(`docker build -t ${IMAGE_NAME} .`); commandLog('Removing container if exists.'); @@ -74,7 +78,7 @@ const execCommand = (command) => { commandLog(`Running container with name ${CONTAINER_NAME}`); await execCommand( - `docker run -d -p 8010:8010 -p 8001:8001 -p 5440:5432 --name ${CONTAINER_NAME} ${IMAGE_NAME}`, + `docker run -d -p 8010:8010 -p 8001:8001 --name ${CONTAINER_NAME} ${IMAGE_NAME}`, ); commandLog('Getting IP of container'); From 36a6117451960d1a0d3148275316e05fdf8a6911 Mon Sep 17 00:00:00 2001 From: Maarten Coppens Date: Sun, 28 Feb 2021 19:37:19 -0300 Subject: [PATCH 7/7] add script to package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a2def19..d73029e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "A full Leasehold node implementation", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "start": "node start.js" }, "repository": { "type": "git",