From f1668ba274bb130d6a837397ae6ebea793035f6d Mon Sep 17 00:00:00 2001 From: Phillip Wirth Date: Fri, 27 Aug 2021 19:52:39 +0200 Subject: [PATCH] pin requirements clean up docker stuff also includes the fix where the webserver is not able to punlish any message --- .github/workflows/build_push_service.yml | 17 ++--- Dockerfile | 13 ++++ Dockerfile.scanfile | 22 ------ Dockerfile.scanfile.20191118 | 23 ------- Dockerfile.scanfile.alpine | 18 ----- Dockerfile.scanurl | 22 ------ Dockerfile.scanurl.20191118 | 23 ------- Dockerfile.webserver | 22 ------ Dockerfile.webserver.20191118 | 23 ------- .../all/antivirus_check_service.yml | 3 +- .../templates/scanfile-deployment.yml.j2 | 3 +- .../templates/web-deployment.yml.j2 | 3 +- antivirus_service/service.py | 13 ++-- antivirus_service/webserver.py | 69 +++++++------------ requirements.txt | 8 +++ resources/encrypt_rabbitmq_password.py | 13 ++-- setup.py | 16 ++--- version | 1 - 18 files changed, 78 insertions(+), 234 deletions(-) create mode 100644 Dockerfile delete mode 100644 Dockerfile.scanfile delete mode 100644 Dockerfile.scanfile.20191118 delete mode 100644 Dockerfile.scanfile.alpine delete mode 100644 Dockerfile.scanurl delete mode 100644 Dockerfile.scanurl.20191118 delete mode 100644 Dockerfile.webserver delete mode 100644 Dockerfile.webserver.20191118 create mode 100644 requirements.txt mode change 100644 => 100755 setup.py delete mode 100644 version diff --git a/.github/workflows/build_push_service.yml b/.github/workflows/build_push_service.yml index 2c308fe..da9d118 100644 --- a/.github/workflows/build_push_service.yml +++ b/.github/workflows/build_push_service.yml @@ -9,18 +9,16 @@ on: - dependabot/* jobs: - build_and_push_docker: + build_and_push: runs-on: ubuntu-latest - strategy: - matrix: - service: [ scanfile, scanurl, webserver ] steps: - uses: actions/checkout@v2 + - name: Docker meta Service Name id: docker_meta_img uses: docker/metadata-action@v3 with: - images: ghcr.io/${{ github.repository }}-${{ matrix.service }} + images: ghcr.io/${{ github.repository }} tags: | type=ref,event=branch,enable=false,priority=600 type=sha,enable=true,priority=600,prefix= @@ -31,23 +29,20 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Build and push ${{ github.repository }} uses: docker/build-push-action@v2 with: context: . - file: ./Dockerfile.${{ matrix.service }} + file: ./Dockerfile platforms: linux/amd64 push: true - tags: ghcr.io/${{ github.repository }}-${{ matrix.service }}:${{ github.sha }} + tags: ghcr.io/${{ github.repository }}:${{ github.sha }} labels: ${{ steps.docker_meta_img.outputs.labels }} dispatch: runs-on: ubuntu-latest needs: - - build_and_push_docker + - build_and_push steps: - uses: actions/checkout@v2 - name: Extract branch name diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..64e9d0d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.7-slim-buster + +ENV LC_ALL=C.UTF-8 +ENV LANG C.UTF-8 + +WORKDIR /usr/src/app +COPY requirements.txt setup.py ./ +RUN pip3 install . +COPY antivirus_service ./antivirus_service + +RUN python3 ./setup.py develop + +ENTRYPOINT ["/usr/local/bin/antivirus"] diff --git a/Dockerfile.scanfile b/Dockerfile.scanfile deleted file mode 100644 index c66cfa0..0000000 --- a/Dockerfile.scanfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM debian:buster-slim - -WORKDIR /usr/src/app - -COPY . . - -RUN apt update -RUN apt install -y python3 python3-pip python3-dev -RUN pip3 install environs -RUN pip3 install setuptools -RUN pip3 install PyYAML -RUN pip3 install pika -RUN pip3 install requests -RUN pip3 install click -RUN pip3 install pyClamd -RUN pip3 install aiohttp -RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "scan-file"] diff --git a/Dockerfile.scanfile.20191118 b/Dockerfile.scanfile.20191118 deleted file mode 100644 index 24e72e3..0000000 --- a/Dockerfile.scanfile.20191118 +++ /dev/null @@ -1,23 +0,0 @@ -FROM debian:stretch - -WORKDIR /usr/src/app - -COPY . . - -RUN apt update \ - && apt install -y python3 \ - python3-dev \ - python3-setuptools \ - python3-pip \ - python3-yaml \ - python3-pika \ - python3-requests \ - python3-click \ - python3-pyclamd \ - python3-aiohttp -RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "scan_file"] \ No newline at end of file diff --git a/Dockerfile.scanfile.alpine b/Dockerfile.scanfile.alpine deleted file mode 100644 index 70f758b..0000000 --- a/Dockerfile.scanfile.alpine +++ /dev/null @@ -1,18 +0,0 @@ -FROM python:3.5-alpine - -WORKDIR /usr/src/app - -COPY . . - -RUN apk update && apk upgrade && apk add --no-cache build-base cairo-dev gobject-introspection-dev libffi-dev openssl-dev && easy_install --upgrade pip && pip install --no-cache-dir --user aio-pika aiohttp "aiormq<3.0" async-timeout chardet click colorama cryptography idna keyring keyrings.alt multidict pamqp pika pyasn1 pyClamd pycrypto pygobject pyxdg PyYAML requests SecretStorage setuptools six typing-extensions urllib3 wheel yarl && python3 ./setup.py develop - -#RUN apk update && apk upgrade && apk add --no-cache build-base cairo-dev gobject-introspection-dev libffi-dev openssl-dev python3-dev py3-setuptools py3-yaml py3-requests && pip install --no-cache-dir aio-pika aiohttp "aiormq<3.0" async-timeout chardet click colorama cryptography idna keyring keyrings.alt multidict pamqp pika pyasn1 pyClamd pycrypto pygobject pyxdg SecretStorage six typing-extensions urllib3 wheel yarl - -#COPY . . - -#RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "scan_file"] diff --git a/Dockerfile.scanurl b/Dockerfile.scanurl deleted file mode 100644 index b50bce1..0000000 --- a/Dockerfile.scanurl +++ /dev/null @@ -1,22 +0,0 @@ -FROM debian:buster-slim - -WORKDIR /usr/src/app - -COPY . . - -RUN apt update -RUN apt install -y python3 python3-pip python3-dev -RUN pip3 install environs -RUN pip3 install setuptools -RUN pip3 install PyYAML -RUN pip3 install pika -RUN pip3 install requests -RUN pip3 install click -RUN pip3 install pyClamd -RUN pip3 install aiohttp -RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "scan-url"] diff --git a/Dockerfile.scanurl.20191118 b/Dockerfile.scanurl.20191118 deleted file mode 100644 index 951a0dc..0000000 --- a/Dockerfile.scanurl.20191118 +++ /dev/null @@ -1,23 +0,0 @@ -FROM debian:stretch - -WORKDIR /usr/src/app - -COPY . . - -RUN apt update \ - && apt install -y python3 \ - python3-dev \ - python3-setuptools \ - python3-pip \ - python3-yaml \ - python3-pika \ - python3-requests \ - python3-click \ - python3-pyclamd \ - python3-aiohttp -RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "scan_url"] \ No newline at end of file diff --git a/Dockerfile.webserver b/Dockerfile.webserver deleted file mode 100644 index 1cdbc8c..0000000 --- a/Dockerfile.webserver +++ /dev/null @@ -1,22 +0,0 @@ -FROM debian:buster-slim - -WORKDIR /usr/src/app - -COPY . . - -RUN apt update -RUN apt install -y python3 python3-pip python3-dev -RUN pip3 install environs -RUN pip3 install setuptools -RUN pip3 install PyYAML -RUN pip3 install pika -RUN pip3 install requests -RUN pip3 install click -RUN pip3 install pyClamd -RUN pip3 install aiohttp -RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "webserver"] diff --git a/Dockerfile.webserver.20191118 b/Dockerfile.webserver.20191118 deleted file mode 100644 index 1776531..0000000 --- a/Dockerfile.webserver.20191118 +++ /dev/null @@ -1,23 +0,0 @@ -FROM debian:stretch - -WORKDIR /usr/src/app - -COPY . . - -RUN apt update \ - && apt install -y python3 \ - python3-dev \ - python3-setuptools \ - python3-pip \ - python3-yaml \ - python3-pika \ - python3-requests \ - python3-click \ - python3-pyclamd \ - python3-aiohttp -RUN python3 ./setup.py develop - -ENV LC_ALL=C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/usr/local/bin/antivirus", "webserver"] \ No newline at end of file diff --git a/ansible/group_vars/all/antivirus_check_service.yml b/ansible/group_vars/all/antivirus_check_service.yml index 7e571c4..ea5f095 100644 --- a/ansible/group_vars/all/antivirus_check_service.yml +++ b/ansible/group_vars/all/antivirus_check_service.yml @@ -1,5 +1,4 @@ --- -ANTIVIRUS_WEBSERVER_IMAGE_NAME: ghcr.io/hpi-schul-cloud/antivirus_check_service-webserver -ANTIVIRUS_SCANFILE_IMAGE_NAME: ghcr.io/hpi-schul-cloud/antivirus_check_service-scanfile +ANTIVIRUS_IMAGE_NAME: ghcr.io/hpi-schul-cloud/antivirus_check_service ANTIVIRUS_PORT: 8080 diff --git a/ansible/roles/antivirus_check_service/templates/scanfile-deployment.yml.j2 b/ansible/roles/antivirus_check_service/templates/scanfile-deployment.yml.j2 index ace755b..bb0a7e9 100644 --- a/ansible/roles/antivirus_check_service/templates/scanfile-deployment.yml.j2 +++ b/ansible/roles/antivirus_check_service/templates/scanfile-deployment.yml.j2 @@ -33,8 +33,9 @@ spec: runAsNonRoot: true containers: - name: antivirus-scanfile - image: {{ ANTIVIRUS_SCANFILE_IMAGE_NAME }}:{{ APP_IMAGE_TAG }} + image: {{ ANTIVIRUS_IMAGE_NAME }}:{{ APP_IMAGE_TAG }} imagePullPolicy: IfNotPresent + args: ["scan-file"] envFrom: - secretRef: name: antivirus-secret diff --git a/ansible/roles/antivirus_check_service/templates/web-deployment.yml.j2 b/ansible/roles/antivirus_check_service/templates/web-deployment.yml.j2 index 12e444c..d8b0a31 100644 --- a/ansible/roles/antivirus_check_service/templates/web-deployment.yml.j2 +++ b/ansible/roles/antivirus_check_service/templates/web-deployment.yml.j2 @@ -33,8 +33,9 @@ spec: runAsNonRoot: true containers: - name: antivirus-webserver - image: {{ ANTIVIRUS_WEBSERVER_IMAGE_NAME }}:{{ APP_IMAGE_TAG }} + image: {{ ANTIVIRUS_IMAGE_NAME }}:{{ APP_IMAGE_TAG }} imagePullPolicy: IfNotPresent + args: ["webserver"] ports: - name: antivirus-web containerPort: {{ ANTIVIRUS_PORT }} diff --git a/antivirus_service/service.py b/antivirus_service/service.py index 50fb668..8db203d 100644 --- a/antivirus_service/service.py +++ b/antivirus_service/service.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- import os -import sys -import yaml import click import logging @@ -17,7 +15,7 @@ def __init__(self, env, debug): self.env = env self.project_root = os.path.dirname(os.path.abspath(__file__)) self.config = {} - + env = Env() with env.prefixed(self.env.upper()+"_"): self.config[self.env]={} @@ -49,7 +47,7 @@ def __init__(self, env, debug): self.config[self.env][param]['scan_url']['queue']=env("QUEUE","scan_url") self.config[self.env][param]['scan_url']['routing_key']=env("ROUTING_KEY","scan_url") - loglevel = logging.INFO if debug else logging.ERROR + loglevel = logging.INFO if debug else logging.ERROR logging.basicConfig(level=loglevel) @@ -96,5 +94,10 @@ def webserver(ctx): except KeyboardInterrupt: webserver.stop() + def main(): - cli(obj={}) \ No newline at end of file + cli(obj={}) + + +if __name__ == "__main__": + main() diff --git a/antivirus_service/webserver.py b/antivirus_service/webserver.py index 6cdb4ae..4b37071 100644 --- a/antivirus_service/webserver.py +++ b/antivirus_service/webserver.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- import json -import time import base64 import logging -import aio_pika -import asyncio + +import pika from aiohttp import web -from pprint import pformat from functools import wraps from antivirus_service.clamd import Clamd @@ -31,45 +29,23 @@ def __init__(self, settings): self.amqp_config = settings.config[settings.env]['amqp'] self.clamd_config = settings.config[settings.env]['clamd'] self.auth_keys = [ - base64.b64encode(bytes(entry, 'utf-8')).decode('ascii') - for entry in settings.config[settings.env]['webserver']['auth_users']] + base64.b64encode(bytes(entry, 'utf-8')).decode('ascii') + for entry in settings.config[settings.env]['webserver']['auth_users']] + + self.clamd = Clamd(self.clamd_config) - def run(self): app = web.Application() app.router.add_get('/', self.index) app.router.add_post('/scan/file', self.handle_file) app.router.add_post('/scan/url', self.handle_uri) app.router.add_get('/antivirus-version', self.handle_version) - - app.on_startup.append(self.on_startup) - app.on_shutdown.append(self.on_shutdown) - self.app = app - web.run_app(app) - self.pika_startup() + + def run(self): + web.run_app(self.app) def stop(self): self.app.loop.close() - self.pika_shutdown() - - async def pika_startup(self): - print("Establish amqp connection and channel") - self.loop_ampq = asyncio.new_event_loop() - loop = self.loop_ampq - self.connection = await aio_pika.connect_robust(self.amqp_config['url'], loop=loop) - self.channel = await self.connection.channel() - self.loop_ampq.run_forever() - - async def pika_shutdown(self): - print("Close amqp connection and channel") - self.loop_ampq.run_until_complete(self.channel.close()) - self.loop_ampq.run_until_complete(self.connection.close()) - - async def on_startup(self, app): - self.clamd = Clamd(self.clamd_config) - - async def on_shutdown(self, app): - pass async def index(self, request): doc = { @@ -120,12 +96,13 @@ async def handle_file(self, request): assert 'download_uri' in payload assert 'callback_uri' in payload - await self.enqueue_scan_file_request(body) + self.enqueue_scan_file_request(body) return web.Response(status=202) except AssertionError: return web.Response(status=422, text='Unprocessable Entity: missing parameter') except Exception as e: + logging.error('error', exc_info=True) return web.Response(status=500, text=str(e)) @auth_required @@ -136,23 +113,27 @@ async def handle_uri(self, request): assert 'url' in payload assert 'callback_uri' in payload - await self.enqueue_scan_url_request(body) - return web.Response(status=202, text='The request has been accepted for processing, but the processing has not been completed.') + self.enqueue_scan_url_request(body) + return web.Response(status=202, + text='The request has been accepted for processing, but the processing has not been completed.') except AssertionError: return web.Response(status=422, text='Unprocessable Entity: missing parameter') except Exception as e: + logging.error('error', exc_info=True) return web.Response(status=500, text=str(e)) - async def enqueue_scan_file_request(self, body): - await self.channel.default_exchange.publish( - aio_pika.Message(body=body), routing_key=self.amqp_config['scan_file']['routing_key'] - ) + def enqueue_scan_file_request(self, body): + self.send(body, self.amqp_config['scan_file']['routing_key']) + + def enqueue_scan_url_request(self, body): + self.send(body, routing_key=self.amqp_config['scan_url']['routing_key']) - async def enqueue_scan_url_request(self, body): - await self.channel.default_exchange.publish( - aio_pika.Message(body=body), routing_key=self.amqp_config['scan_url']['routing_key'] - ) + def send(self, body, routing_key): + params = pika.URLParameters(self.amqp_config['url']) + with pika.BlockingConnection(params) as con: + with con.channel() as channel: + channel.basic_publish(body=body, exchange='', routing_key=routing_key) @auth_required async def handle_version(self, request): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..051bf3c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +requests==2.26.0 +click==8.0.1 +pyClamd==0.4.0 +aiohttp==3.7.4.post0 +PyYAML==5.4.1 +pika==1.2.0 +aio_pika==7.0.0b3 +environs==9.3.3 \ No newline at end of file diff --git a/resources/encrypt_rabbitmq_password.py b/resources/encrypt_rabbitmq_password.py index f02bac7..2b54272 100644 --- a/resources/encrypt_rabbitmq_password.py +++ b/resources/encrypt_rabbitmq_password.py @@ -36,6 +36,7 @@ def generate_random_salt(): # only 4 Bytes = 32 Bit return ''.join(random.choice(chars) for i in range(4)).encode('hex') + def encode_rabbit_password_hash(salt, password): salt_and_password = salt + password.encode('utf-8').encode('hex') salt_and_password = bytearray.fromhex(salt_and_password) @@ -44,10 +45,12 @@ def encode_rabbit_password_hash(salt, password): password_hash = bytearray.fromhex(salt + salted_sha256) return binascii.b2a_base64(password_hash).strip() + def decode_rabbit_password_hash(password_hash): password_hash = binascii.a2b_base64(password_hash) decoded_hash = password_hash.encode('hex') - return (decoded_hash[0:8], decoded_hash[8:]) + return decoded_hash[0:8], decoded_hash[8:] + def check_rabbit_password(test_password, password_hash): salt, _ = decode_rabbit_password_hash(password_hash) @@ -62,7 +65,7 @@ def check_rabbit_password(test_password, password_hash): salt = generate_random_salt() sha256hash = encode_rabbit_password_hash(salt, args.password) -print '== Your generated sha256 hash ==' -print '%s (with salt: %s)' % (sha256hash, salt) -print -print '== Check: ', check_rabbit_password(args.password, sha256hash) \ No newline at end of file +print('== Your generated sha256 hash ==') +print('%s (with salt: %s)' % (sha256hash, salt)) +print() +print('== Check: ', check_rabbit_password(args.password, sha256hash)) diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 8386204..22c451a --- a/setup.py +++ b/setup.py @@ -2,22 +2,16 @@ # -*- coding: utf-8 -*- from setuptools import setup, find_packages -def set_version(): - path = './version' - version_file = open(path,'r') - get_version = version_file.read() - version_file.close() - return get_version +with open("requirements.txt", "r") as fh: + requirements = fh.readlines() setup( name='antivirus_service', - version=set_version(), + version='2.0.0', description='This service detects virus in downloaded files by using clamd', - author='matthias wiesner', + author='hpi schul-cloud', packages=find_packages(exclude='tests'), - install_requires=[ - 'aio_pika' - ], + install_requires=[req for req in requirements if req[:2] != "# "], entry_points={ 'console_scripts': [ 'antivirus=antivirus_service.service:main' diff --git a/version b/version deleted file mode 100644 index afaf360..0000000 --- a/version +++ /dev/null @@ -1 +0,0 @@ -1.0.0 \ No newline at end of file