diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 69a8ebf..a010ff1 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -52,7 +52,7 @@ jobs: # username: ${{ secrets.JFROG_USERNAME }} # password: ${{ secrets.JFROG_TOKEN }} - - name: Build and push + - name: Build and push (pewpew) uses: docker/build-push-action@v4 with: context: . @@ -60,6 +60,15 @@ jobs: push: true tags: | sjultra/pewpew:latest + + - name: Build and push (adapter) + uses: docker/build-push-action@v4 + with: + context: ./docker/datasource + file: ./docker/datasource/Dockerfile + push: true + tags: | + sjultra/pewpew-adapter:latest # ghcr.io/${{ github.repository_owner }}/pewpew:latest # install and authenticate jfrog/artifactory CLI diff --git a/docker/Dockerfile b/docker/Dockerfile index 7c16640..fade157 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,5 +9,9 @@ RUN mkdir -p /opt/pewpew VOLUME ./data +# init external data file +RUN touch ./data/data.json +RUN echo "{\"data\": []}" > ./data/data.json + CMD ["http-server","-a","0.0.0.0","-p","8080","/opt/pewpew"] diff --git a/docker/datasource/package-lock.json b/docker/datasource/package-lock.json index 2f1e3ac..20ef1e6 100644 --- a/docker/datasource/package-lock.json +++ b/docker/datasource/package-lock.json @@ -8,9 +8,8 @@ "name": "pewpew-datasource", "version": "1.0.0", "license": "ISC", - "devDependencies": { + "dependencies": { "@types/node": "^20.8.7", - "node-fetch": "^3.3.2", "ts-node": "^10.9.1", "typescript": "^5.2.2" } @@ -19,7 +18,6 @@ "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, "dependencies": { "@jridgewell/trace-mapping": "0.3.9" }, @@ -31,7 +29,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", - "dev": true, "engines": { "node": ">=6.0.0" } @@ -39,14 +36,12 @@ "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.9", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" @@ -55,32 +50,27 @@ "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", - "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", - "dev": true + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==" }, "node_modules/@tsconfig/node12": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==" }, "node_modules/@tsconfig/node14": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==" }, "node_modules/@tsconfig/node16": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==" }, "node_modules/@types/node": { "version": "20.8.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.7.tgz", "integrity": "sha512-21TKHHh3eUHIi2MloeptJWALuCu5H7HQTdTrWIFReA8ad+aggoX+lRes3ex7/FtpC+sVUpFMQ+QTfYr74mruiQ==", - "dev": true, "dependencies": { "undici-types": "~5.25.1" } @@ -89,7 +79,6 @@ "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", - "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -101,7 +90,6 @@ "version": "8.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -109,116 +97,30 @@ "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==" }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "dev": true, - "engines": { - "node": ">= 12" - } + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true, "engines": { "node": ">=0.3.1" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "dev": true, - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "dev": true, - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" }, "node_modules/ts-node": { "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "dev": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -261,7 +163,6 @@ "version": "5.2.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", - "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -273,29 +174,17 @@ "node_modules/undici-types": { "version": "5.25.3", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.25.3.tgz", - "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==", - "dev": true + "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==" }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true - }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "dev": true, - "engines": { - "node": ">= 8" - } + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==" }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true, "engines": { "node": ">=6" } diff --git a/docker/datasource/package.json b/docker/datasource/package.json index c86d57f..7481483 100644 --- a/docker/datasource/package.json +++ b/docker/datasource/package.json @@ -8,7 +8,7 @@ }, "author": "", "license": "ISC", - "devDependencies": { + "dependencies": { "@types/node": "^20.8.7", "ts-node": "^10.9.1", "typescript": "^5.2.2" diff --git a/docker/datasource/src/index.ts b/docker/datasource/src/index.ts index 3ce6d66..b5f6d2b 100644 --- a/docker/datasource/src/index.ts +++ b/docker/datasource/src/index.ts @@ -29,8 +29,7 @@ async function main() { const signIns = await getSignIns(accessToken) - const datapoints = await traceSignIns(signIns) - console.log(datapoints); + const datapoints = await traceSignIns(signIns, false) await writeToDisk({data: datapoints}) } @@ -111,18 +110,24 @@ async function traceSignIns(signIns: SignIn[], smoke: boolean = true): Promise= LOOKUP_IP_LIMIT) { - if (smoke) break + if (smoke) { + console.log("[SMOKE] smoke run completed."); + break + } + console.log("[SLEEP] 62500ms/62.5s"); await sleep(62500) counter = 0 iteration += 1 } + + if (counter === 0) { + console.log(`Fetching IP info for ${LOOKUP_IP_LIMIT} IP addresses (batch ${iteration}/${Math.ceil(signIns.length/LOOKUP_IP_LIMIT)})`); + + } + + console.log(`Processing ${counter+1}/${LOOKUP_IP_LIMIT}`); + const origin = await lookupIP(ip) const datapoint = { hit: { diff --git a/helm/service/templates/datasource.yaml b/helm/service/templates/datasource.yaml new file mode 100644 index 0000000..af3d669 --- /dev/null +++ b/helm/service/templates/datasource.yaml @@ -0,0 +1,47 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: datasource +spec: + schedule: {{ .Values.datasource.schedule }} + successfulJobsHistoryLimit: 0 + jobTemplate: + spec: + selector: + matchLabels: + app: datasource + template: + metadata: + labels: + app: datasource + spec: + containers: + - name: datasource + image: "{{ .Values.datasource.image }}:{{ .Values.datasource.tag }}" + + volumeMounts: + - name: {{ .Values.adapter.name }} + mountPath: /datasource/data + # args: + # - sh + # - -c + # - | + # sleep 2 + # "/opt/myapp/myscript.sh" + # sleep 30 + # env: + # - name: APP_SECRET + # valueFrom: + # secretKeyRef: + # name: {{ .Values.appsecret }} + # key: app_secret + # - name: APP_CLIENT_ID + # valueFrom: + # secretKeyRef: + # name: {{ .Values.appsecret }} + # key: app_client_id + # ports: + # - containerPort: {{ .Values.service.internalPort }} + # restartPolicy: {{ .Values.restartPolicy }} + # imagePullSecrets: + # - name: artifactory \ No newline at end of file diff --git a/helm/service/templates/pewpew.yaml b/helm/service/templates/pewpew.yaml index 5a38791..63539f3 100644 --- a/helm/service/templates/pewpew.yaml +++ b/helm/service/templates/pewpew.yaml @@ -15,6 +15,10 @@ spec: containers: - name: pewpew image: "{{ .Values.pewpew.image }}:{{ .Values.pewpew.tag }}" + + volumeMounts: + - name: {{ .Values.adapter.name }} + mountPath: /opt/pewpew/data --- apiVersion: v1 kind: Service diff --git a/helm/service/templates/pvc.yaml b/helm/service/templates/pvc.yaml new file mode 100644 index 0000000..eb9b29d --- /dev/null +++ b/helm/service/templates/pvc.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.adapter.name }} +spec: + storageClassName: "" # Empty string must be explicitly set otherwise default StorageClass will be set + volumeName: {{ .Values.adapter.name }} \ No newline at end of file diff --git a/helm/service/values.yaml b/helm/service/values.yaml index b292106..871162f 100644 --- a/helm/service/values.yaml +++ b/helm/service/values.yaml @@ -7,3 +7,11 @@ pewpew: tag: latest httpPort: 80 targetPort: 8080 + +adapter: + image: sjultra/pewpew-adapter + tag: latest + schedule: '"0 * * * *"' + +pvc: + name: pewpew-adapter-pvc