From 92dbc04803025e3cf2f90c22ab77325b4452996e Mon Sep 17 00:00:00 2001 From: Zoey Date: Tue, 7 Jan 2025 10:50:53 +0100 Subject: [PATCH] improve host recreation --- Caddy.Dockerfile | 2 +- Dockerfile | 10 +-- backend/internal/access-list.js | 14 ++-- backend/internal/certificate.js | 2 +- backend/internal/host.js | 4 +- backend/internal/nginx.js | 29 ++----- backend/package.json | 2 +- backend/setup.js | 133 +++++++++++++++----------------- rootfs/usr/local/bin/launch.sh | 1 + rootfs/usr/local/bin/start.sh | 4 +- 10 files changed, 90 insertions(+), 111 deletions(-) diff --git a/Caddy.Dockerfile b/Caddy.Dockerfile index 7c73154c9..65b40bca9 100644 --- a/Caddy.Dockerfile +++ b/Caddy.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.21.0 +FROM alpine:3.21.1 RUN apk add --no-cache ca-certificates tzdata COPY --from=caddy:2.8.4 /usr/bin/caddy /usr/bin/caddy COPY Caddyfile /etc/caddy/Caddyfile diff --git a/Dockerfile b/Dockerfile index 510e00a72..0aa813e9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:labs -FROM --platform="$BUILDPLATFORM" alpine:3.21.0 AS frontend +FROM --platform="$BUILDPLATFORM" alpine:3.21.1 AS frontend SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG NODE_ENV=production \ NODE_OPTIONS=--openssl-legacy-provider @@ -19,7 +19,7 @@ COPY darkmode.css /app/dist/css/darkmode.css COPY security.txt /app/dist/.well-known/security.txt -FROM --platform="$BUILDPLATFORM" alpine:3.21.0 AS build-backend +FROM --platform="$BUILDPLATFORM" alpine:3.21.1 AS build-backend SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG NODE_ENV=production \ TARGETARCH @@ -38,7 +38,7 @@ RUN apk upgrade --no-cache -a && \ fi && \ yarn cache clean --all && \ clean-modules --yes -FROM alpine:3.21.0 AS strip-backend +FROM alpine:3.21.1 AS strip-backend COPY --from=build-backend /app /app RUN apk upgrade --no-cache -a && \ apk add --no-cache ca-certificates binutils file && \ @@ -46,7 +46,7 @@ RUN apk upgrade --no-cache -a && \ find /app/node_modules -name "*.node" -type f -exec file {} \; -FROM --platform="$BUILDPLATFORM" alpine:3.21.0 AS crowdsec +FROM --platform="$BUILDPLATFORM" alpine:3.21.1 AS crowdsec SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG CSNB_VER=v1.0.8 WORKDIR /src @@ -72,7 +72,7 @@ RUN apk upgrade --no-cache -a && \ sed -i "s|APPSEC_PROCESS_TIMEOUT=.*|APPSEC_PROCESS_TIMEOUT=10000|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf -FROM zoeyvid/nginx-quic:371-python +FROM zoeyvid/nginx-quic:372-python SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG CRS_VER=v4.10.0 COPY rootfs / diff --git a/backend/internal/access-list.js b/backend/internal/access-list.js index 9f1f31a50..97e9fa4c8 100644 --- a/backend/internal/access-list.js +++ b/backend/internal/access-list.js @@ -86,7 +86,7 @@ const internalAccessList = { .build(row) .then(() => { if (row.proxy_host_count) { - return internalNginx.bulkGenerateConfigs('proxy_host', row.proxy_hosts); + return internalNginx.bulkGenerateConfigs(proxyHostModel, 'proxy_host', row.proxy_hosts); } }) .then(() => { @@ -163,7 +163,7 @@ const internalAccessList = { return query.then(() => { // Add new items - if (promises.length) { + if (promises.length > 0) { return Promise.all(promises); } }); @@ -190,7 +190,7 @@ const internalAccessList = { return query.then(() => { // Add new items - if (promises.length) { + if (promises.length > 0) { return Promise.all(promises); } }); @@ -221,7 +221,7 @@ const internalAccessList = { .build(row) .then(() => { if (row.proxy_host_count) { - return internalNginx.bulkGenerateConfigs('proxy_host', row.proxy_hosts); + return internalNginx.bulkGenerateConfigs(proxyHostModel, 'proxy_host', row.proxy_hosts); } }) .then(internalNginx.reload) @@ -320,7 +320,7 @@ const internalAccessList = { row.proxy_hosts[idx].access_list_id = 0; }); - return internalNginx.bulkGenerateConfigs('proxy_host', row.proxy_hosts); + return internalNginx.bulkGenerateConfigs(proxyHostModel, 'proxy_host', row.proxy_hosts); }) .then(() => { return internalNginx.reload(); @@ -476,12 +476,12 @@ const internalAccessList = { } }).then((htpasswd_file) => { // 3. generate password for each user - if (list.items.length) { + if (list.items.length > 0) { return new Promise((resolve, reject) => { batchflow(list.items) .sequential() .each((i, item, next) => { - if (typeof item.password !== 'undefined' && item.password.length) { + if (typeof item.password !== 'undefined' && item.password.length > 0) { logger.info('Adding: ' + item.username); try { diff --git a/backend/internal/certificate.js b/backend/internal/certificate.js index 07870df27..8f0eb05de 100644 --- a/backend/internal/certificate.js +++ b/backend/internal/certificate.js @@ -60,7 +60,7 @@ const internalCertificate = { .where('is_deleted', 0) .andWhere('provider', 'letsencrypt') .then((certificates) => { - if (certificates && certificates.length) { + if (certificates && certificates.length > 0) { const promises = []; certificates.map(function (certificate) { diff --git a/backend/internal/host.js b/backend/internal/host.js index b858955bb..52396507b 100644 --- a/backend/internal/host.js +++ b/backend/internal/host.js @@ -160,7 +160,7 @@ const internalHost = { _checkHostnameRecordsTaken: function (hostname, existing_rows, ignore_id) { let is_taken = false; - if (existing_rows && existing_rows.length) { + if (existing_rows && existing_rows.length > 0) { existing_rows.map(function (existing_row) { existing_row.domain_names.map(function (existing_hostname) { // Does this domain match? @@ -186,7 +186,7 @@ const internalHost = { _getHostsWithDomains: function (hosts, domain_names) { const response = []; - if (hosts && hosts.length) { + if (hosts && hosts.length > 0) { hosts.map(function (host) { let host_matches = false; diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 51fd6a37f..73d186d51 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -80,22 +80,12 @@ const internalNginx = { reload: () => { if (process.env.ACME_OCSP_STAPLING === 'true') { return utils.execFile('certbot-ocsp-fetcher.sh', ['-c', '/data/tls/certbot', '-o', '/data/tls/certbot/live', '--no-reload-webserver', '--quiet']).finally(() => { - if (fs.existsSync('/usr/local/nginx/logs/nginx.pid') && fs.readFileSync('/usr/local/nginx/logs/nginx.pid', 'utf8').trim().length > 0) { - logger.info('Reloading Nginx'); - return utils.execFile('nginx', ['-s', 'reload']); - } else { - logger.info('Starting Nginx'); - utils.execfg('nginx', ['-e', 'stderr']); - } - }); - } else { - if (fs.existsSync('/usr/local/nginx/logs/nginx.pid') && fs.readFileSync('/usr/local/nginx/logs/nginx.pid', 'utf8').trim().length > 0) { logger.info('Reloading Nginx'); return utils.execFile('nginx', ['-s', 'reload']); - } else { - logger.info('Starting Nginx'); - utils.execfg('nginx', ['-e', 'stderr']); - } + }); + } else { + logger.info('Reloading Nginx'); + return utils.execFile('nginx', ['-s', 'reload']); } }, @@ -276,13 +266,10 @@ const internalNginx = { * @param {Array} hosts * @returns {Promise} */ - bulkGenerateConfigs: (host_type, hosts) => { - const promises = []; - hosts.map(function (host) { - promises.push(internalNginx.generateConfig(host_type, host)); - }); - - return Promise.all(promises); + bulkGenerateConfigs: (model, host_type, hosts) => { + return hosts.reduce((promise, host) => { + return promise.then(() => internalNginx.configure(model, host_type, host)); + }, Promise.resolve()); }, /** diff --git a/backend/package.json b/backend/package.json index 974c87425..b98c919df 100644 --- a/backend/package.json +++ b/backend/package.json @@ -30,7 +30,7 @@ "author": "Jamie Curnow and ZoeyVid ", "license": "MIT", "devDependencies": { - "@apidevtools/swagger-parser": "10.1.0", + "@apidevtools/swagger-parser": "10.1.1", "@eslint/js": "9.17.0", "eslint": "9.17.0", "eslint-config-prettier": "9.1.0", diff --git a/backend/setup.js b/backend/setup.js index 8070203a5..4df96cad7 100644 --- a/backend/setup.js +++ b/backend/setup.js @@ -101,6 +101,15 @@ const setupDefaultSettings = () => { logger.info('Default settings added'); }); } + }) + .then(() => { + settingModel + .query() + .where('id', 'default-site') + .first() + .then((row) => { + internalNginx.generateConfig('default', row); + }); }); }; @@ -115,7 +124,7 @@ const setupCertbotPlugins = () => { .where('is_deleted', 0) .andWhere('provider', 'letsencrypt') .then((certificates) => { - if (certificates && certificates.length) { + if (certificates && certificates.length > 0) { const plugins = []; const promises = []; @@ -129,7 +138,7 @@ const setupCertbotPlugins = () => { }); return certbot.installPlugins(plugins).then(() => { - if (promises.length) { + if (promises.length > 0) { return Promise.all(promises).then(() => { logger.info('Added Certbot plugins ' + plugins.join(', ')); }); @@ -145,76 +154,56 @@ const setupCertbotPlugins = () => { * @returns {Promise} */ const regenerateAllHosts = () => { - settingModel - .query() - .where('id', 'default-site') - .first() - .then((row) => { - internalNginx.generateConfig('default', row); - }) - .then(() => { - if (process.env.REGENERATE_ALL === 'true') { - const promises = []; - - promises.push( - proxyModel - .query() - .where('is_deleted', 0) - .andWhere('enabled', 1) - .withGraphFetched('[access_list.[clients, items], certificate]') - .then((rows) => { - if (rows && rows.length) { - internalNginx.bulkGenerateConfigs('proxy_host', rows); - } - }), - ); - - promises.push( - redirectModel - .query() - .where('is_deleted', 0) - .andWhere('enabled', 1) - .withGraphFetched('[certificate]') - .then((rows) => { - if (rows && rows.length) { - internalNginx.bulkGenerateConfigs('redirection_host', rows); - } - }), - ); - - promises.push( - deadModel - .query() - .where('is_deleted', 0) - .andWhere('enabled', 1) - .withGraphFetched('[certificate]') - .then((rows) => { - if (rows && rows.length) { - internalNginx.bulkGenerateConfigs('dead_host', rows); - } - }), - ); - - promises.push( - streamModel - .query() - .where('is_deleted', 0) - .andWhere('enabled', 1) - .then((rows) => { - if (rows && rows.length) { - internalNginx.bulkGenerateConfigs('stream', rows); - } - }), - ); - - // Execute all promises and then write the hash - return Promise.all(promises).then(() => { - utils.writeHash(); - }); - } - }); - - return Promise.resolve(); // Return resolved promise if REGENERATE_ALL is not true + if (process.env.REGENERATE_ALL === 'true') { + return proxyModel + .query() + .where('is_deleted', 0) + .andWhere('enabled', 1) + .withGraphFetched('[access_list.[clients, items], certificate]') + .then((rows) => { + if (rows && rows.length > 0) { + internalNginx.bulkGenerateConfigs(proxyModel, 'proxy_host', rows); + } + }) + .then(() => { + return redirectModel + .query() + .where('is_deleted', 0) + .andWhere('enabled', 1) + .withGraphFetched('[certificate]') + .then((rows) => { + if (rows && rows.length > 0) { + internalNginx.bulkGenerateConfigs(redirectModel, 'redirection_host', rows); + } + }); + }) + .then(() => { + return deadModel + .query() + .where('is_deleted', 0) + .andWhere('enabled', 1) + .withGraphFetched('[certificate]') + .then((rows) => { + if (rows && rows.length > 0) { + internalNginx.bulkGenerateConfigs(deadModel, 'dead_host', rows); + } + }); + }) + .then(() => { + return streamModel + .query() + .where('is_deleted', 0) + .andWhere('enabled', 1) + .then((rows) => { + if (rows && rows.length > 0) { + internalNginx.bulkGenerateConfigs(streamModel, 'stream', rows); + } + }); + }) + .then(() => { + utils.writeHash(); + }); + } }; module.exports = function () { diff --git a/rootfs/usr/local/bin/launch.sh b/rootfs/usr/local/bin/launch.sh index ac6b21e4d..433354142 100755 --- a/rootfs/usr/local/bin/launch.sh +++ b/rootfs/usr/local/bin/launch.sh @@ -84,5 +84,6 @@ if [ "$LOGROTATE" = "true" ]; then while true; do touch /data/logrotate.lock; lo # shellcheck disable=SC2086 if [ "$GOA" = "true" ]; then while true; do if [ -f /data/nginx/access.log ] && [ ! -f /data/logrotate.lock ]; then goaccess --no-global-config --num-tests=0 --tz="$TZ" --date-format="%d/%b/%Y" --time-format="%H:%M:%S" --log-format='[%d:%t %^] %v %h %T "%r" %s %b %b %R %u' --no-ip-validation \ --addr=127.0.0.1 --port="$GOAIWSP" -f /data/nginx/access.log --real-time-html -o /tmp/goa/index.html --persist --restore --db-path=/data/goaccess/data -b /etc/goaccess/browsers.list -b /etc/goaccess/podcast.list $GOACLA; else sleep 10s; fi; done; fi & +nginx -e stderr & aio.sh & index.js diff --git a/rootfs/usr/local/bin/start.sh b/rootfs/usr/local/bin/start.sh index b7a1a314a..2227114fe 100755 --- a/rootfs/usr/local/bin/start.sh +++ b/rootfs/usr/local/bin/start.sh @@ -585,7 +585,8 @@ touch /data/modsecurity/modsecurity-extra.conf \ /data/custom_nginx/stream_top.conf \ /data/custom_nginx/server_stream.conf \ /data/custom_nginx/server_stream_tcp.conf \ - /data/custom_nginx/server_stream_udp.conf + /data/custom_nginx/server_stream_udp.conf/data/crowdsec/crowdsec.conf + if [ -s /data/keys.json ]; then @@ -614,6 +615,7 @@ fi if [ -n "$(ls -A /data/etc 2> /dev/null)" ]; then mv -vn /data/etc/* /data fi +sed -i "s|/data/etc|/data|g" /data/crowdsec/crowdsec.conf #tmp if [ -n "$(ls -A /data/npm 2> /dev/null)" ]; then