wip: Upgrade NodeJS to v16 and krawler to latest version #23

kalisio · Jun 8, 2022 · d7e2c19 · d7e2c19
1 parent 5d5f770
commit d7e2c19
Show file tree

Hide file tree

Showing 9 changed files with 60 additions and 32 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,14 +1,14 @@
 language: node_js
 node_js:
-  - '12'
+  - '16'
 services:
   - docker
 
 notifications:
   email: false
   slack:
     rooms:
-      secure: WuuqaxaQDy7IBzwfR/kJHLce5TOkGGlTxZ5nOG+S6KaG1hnIk/Xnaewp+mijmrsrSzcKzhhCdkF6Bq/Ib/uC76PpwYdgMlP5UCVnd41C3P8atuDdjeHwE4d9u7jAGkKPEmhVzescFef67B9YZJZR2Z4QOG+4GPDMOti/lyBpHYvyTELZHYQoCWYSXJaFkSvzHNL2QGPICsNVtYxagdejCr9dC6c2D37UUBy3Cew5YIqj1FPaV1WTWy5VE79UwcufjiYy3SKBvw7YN0skCmloDg+2esOFr5QHZgkzAHmiXGsSM4FxGfE1c2TpvSMNFbaZkgDUl6BY+TDtrORIUiUjpb6skGNJgP4deSSLNTZXGcnw3DdQRwNHUO229TRYscdAN3HGsNFD/6rwL3UQMP/UWfjv0YDGirfd19gVpuplo4QgQonCgURva3E7RQrSDtGhzMj7U1IqnHDDsQz3JQYsCsRuXiR+wwoceBozbPr7ue+LtqAlDa+CM8mkTKB6QL0mW8lDlQnIfQ/OLGmHFYu9knab/LOvMYwfHscPxjlSxtednj/KigOjJq7XiLugmLG6QG7nfSVfvAZTgiD+MzBOyxlzECVVlSTLV2YOV87Y1VdvtVcLqD7nkBNUMop+SYH25N/+/65tBX4cvMnMg8ljPXuAPVOQrtZNXFHnyp5Da1Y=
+      - secure: WuuqaxaQDy7IBzwfR/kJHLce5TOkGGlTxZ5nOG+S6KaG1hnIk/Xnaewp+mijmrsrSzcKzhhCdkF6Bq/Ib/uC76PpwYdgMlP5UCVnd41C3P8atuDdjeHwE4d9u7jAGkKPEmhVzescFef67B9YZJZR2Z4QOG+4GPDMOti/lyBpHYvyTELZHYQoCWYSXJaFkSvzHNL2QGPICsNVtYxagdejCr9dC6c2D37UUBy3Cew5YIqj1FPaV1WTWy5VE79UwcufjiYy3SKBvw7YN0skCmloDg+2esOFr5QHZgkzAHmiXGsSM4FxGfE1c2TpvSMNFbaZkgDUl6BY+TDtrORIUiUjpb6skGNJgP4deSSLNTZXGcnw3DdQRwNHUO229TRYscdAN3HGsNFD/6rwL3UQMP/UWfjv0YDGirfd19gVpuplo4QgQonCgURva3E7RQrSDtGhzMj7U1IqnHDDsQz3JQYsCsRuXiR+wwoceBozbPr7ue+LtqAlDa+CM8mkTKB6QL0mW8lDlQnIfQ/OLGmHFYu9knab/LOvMYwfHscPxjlSxtednj/KigOjJq7XiLugmLG6QG7nfSVfvAZTgiD+MzBOyxlzECVVlSTLV2YOV87Y1VdvtVcLqD7nkBNUMop+SYH25N/+/65tBX4cvMnMg8ljPXuAPVOQrtZNXFHnyp5Da1Y=
     on_success: always
     on_failure: always
 
@@ -17,6 +17,9 @@ env:
   - JOB=observations
   - JOB=predictions
 
+before_script:
+  - docker login -u="$DOCKER_USER" -p="$DOCKER_PASSWORD"
+
 script:
   - |
    REPO_SLUG="$TRAVIS_REPO_SLUG"
@@ -30,9 +33,6 @@ script:
    fi
    docker build --build-arg KRAWLER_TAG=$KRAWLER_TAG -f dockerfile."$JOB" -t $IMAGE_NAME:"$JOB"-$IMAGE_TAG .
 
-before_script:
-  - docker login -u="$DOCKER_USER" -p="$DOCKER_PASSWORD"
-
 deploy:
   provider: script
   script: docker push $IMAGE_NAME:"$JOB"-$IMAGE_TAG

diff --git a/dockerfile.observations b/dockerfile.observations
@@ -8,14 +8,18 @@ FROM kalisio/krawler:${KRAWLER_TAG} AS krawler
 #
 # Make the job image using the krawler image alias
 #
-FROM node:12-buster-slim
+FROM node:16-buster-slim
 LABEL maintainer="Kalisio <[email protected]>"
 
 ENV CRON="0 */15 * * * *"
 
 # Copy Krawler from the Krawler image alias
 COPY --from=krawler /opt/krawler /opt/krawler
-RUN cd /opt/krawler && yarn link && yarn link @kalisio/krawler
+WORKDIR /opt/krawler
+RUN yarn link && yarn link @kalisio/krawler
+
+# Required as yarn does not seem to set it correctly
+RUN chmod u+x /usr/local/bin/krawler
 
 # Install the job
 COPY jobfile-observations.js .
@@ -24,5 +28,4 @@ COPY jobfile-observations.js .
 HEALTHCHECK --interval=1m --timeout=10s --start-period=1m CMD node /opt/krawler/healthcheck.js
 
 # Run the job
-ENV NODE_PATH=/opt/krawler/node_modules
-CMD node /opt/krawler --cron "$CRON" jobfile-observations.js
+CMD krawler --cron "$CRON" jobfile-observations.js
diff --git a/dockerfile.predictions b/dockerfile.predictions
@@ -8,14 +8,18 @@ FROM kalisio/krawler:${KRAWLER_TAG} AS krawler
 #
 # Make the job image using the krawler image alias
 #
-FROM node:12-buster-slim
+FROM node:16-buster-slim
 LABEL maintainer="Kalisio <[email protected]>"
 
 ENV CRON="0 0 * * * *"
 
 # Copy Krawler from the Krawler image alias
 COPY --from=krawler /opt/krawler /opt/krawler
-RUN cd /opt/krawler && yarn link && yarn link @kalisio/krawler
+WORKDIR /opt/krawler
+RUN yarn link && yarn link @kalisio/krawler
+
+# Required as yarn does not seem to set it correctly
+RUN chmod u+x /usr/local/bin/krawler
 
 # Install the job
 COPY jobfile-predictions.js .
@@ -24,5 +28,4 @@ COPY jobfile-predictions.js .
 HEALTHCHECK --interval=1m --timeout=10s --start-period=1m CMD node /opt/krawler/healthcheck.js
 
 # Run the job
-ENV NODE_PATH=/opt/krawler/node_modules
-CMD node /opt/krawler --run --cron "$CRON" jobfile-predictions.js
+CMD krawler --run --cron "$CRON" jobfile-predictions.js
diff --git a/dockerfile.stations b/dockerfile.stations
@@ -8,14 +8,18 @@ FROM kalisio/krawler:${KRAWLER_TAG} AS krawler
 #
 # Make the job image using the krawler image alias
 #
-FROM node:12-buster-slim
+FROM node:16-buster-slim
 LABEL maintainer="Kalisio <[email protected]>"
 
 ENV CRON="0 0 0 * * *"
 
 # Copy Krawler from the Krawler image alias
 COPY --from=krawler /opt/krawler /opt/krawler
-RUN cd /opt/krawler && yarn link && yarn link @kalisio/krawler
+WORKDIR /opt/krawler
+RUN yarn link && yarn link @kalisio/krawler
+
+# Required as yarn does not seem to set it correctly
+RUN chmod u+x /usr/local/bin/krawler
 
 # Install the job
 COPY jobfile-stations.js .
@@ -24,5 +28,4 @@ COPY jobfile-stations.js .
 HEALTHCHECK --interval=1m --timeout=10s --start-period=1m CMD node /opt/krawler/healthcheck.js
 
 # Run the job
-ENV NODE_PATH=/opt/krawler/node_modules
-CMD node /opt/krawler --run --cron "$CRON" jobfile-stations.js
+CMD krawler --run --cron "$CRON" jobfile-stations.js
diff --git a/jobfile-observations.js b/jobfile-observations.js
@@ -1,6 +1,10 @@
-const krawler = require('@kalisio/krawler')
-const hooks = krawler.hooks
-const _ = require('lodash')
+import _ from 'lodash'
+import { hooks } from '@kalisio/krawler'
+import { fileURLToPath } from 'url'
+import { dirname } from 'path'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
 
 // Configuration
 const dbUrl = process.env.DB_URL || 'mongodb://127.0.0.1:27017/hubeau'
@@ -33,7 +37,7 @@ let generateTasks = (options) => {
 }
 hooks.registerHook('generateTasks', generateTasks)
 
-module.exports = {
+export default {
   id: 'hubeau-observations',
   store: 'fs',
   options: {

diff --git a/jobfile-predictions.js b/jobfile-predictions.js
@@ -1,9 +1,11 @@
-const krawler = require('@kalisio/krawler')
-const hooks = krawler.hooks
-const moment = require('moment')
-const fs = require('fs')
-const path = require('path')
-const _ = require('lodash')
+import _ from 'lodash'
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { hooks } from '@kalisio/krawler'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
 
 // Configuration
 const dbUrl = process.env.DB_URL || 'mongodb://127.0.0.1:27017/kano'
@@ -37,7 +39,7 @@ let generateTasks = (options) => {
 }
 hooks.registerHook('generateTasks', generateTasks)
 
-module.exports = {
+export default {
   id: 'hubeau-predictions',
   store: 'fs',
   options: {

diff --git a/jobfile-stations.js b/jobfile-stations.js
@@ -1,8 +1,13 @@
-const _ = require('lodash')
+import _ from 'lodash'
+import { fileURLToPath } from 'url'
+import { dirname } from 'path'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
 
 const dbUrl = process.env.DB_URL || 'mongodb://127.0.0.1:27017/hubeau'
 
-module.exports = {
+export default {
   id: 'hubeau-stations',
   store: 'fs',
   options: {

diff --git a/package.json b/package.json
@@ -3,6 +3,7 @@
   "description": "",
   "version": "1.2.0",
   "homepage": "https://github.com/kalisio/k-hubeau",
+  "type": "module",
   "keywords": [
     "krawler",
     "kargo",
@@ -31,11 +32,13 @@
     "release:patch": "npm run changelog -- -p && git add CHANGELOG.md && git commit -m \"chore: Updated CHANGELOG.md [skip ci]\" && git push && npm version --force patch",
     "postversion": "git push origin --tags"
   },
-  "dependencies": {},
+  "dependencies": {
+    "lodash": "^4.17.21"
+  },
   "devDependencies": {
     "generate-changelog": "^1.8.0"
   },
   "peerDependencies": {
-    "@kalisio/krawler": "1.1.0"
+    "@kalisio/krawler": "2.0.0"
   }
 }
diff --git a/yarn.lock b/yarn.lock
@@ -25,3 +25,8 @@ github-url-from-git@^1.4.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/github-url-from-git/-/github-url-from-git-1.5.0.tgz#f985fedcc0a9aa579dc88d7aff068d55cc6251a0"
   integrity sha1-+YX+3MCpqledyI16/waNVcxiUaA=
+
+lodash@^4.17.21:
+  version "4.17.21"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
+  integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==