From 4f4673a7ba08051be184b2e08acbeb8237a8edf4 Mon Sep 17 00:00:00 2001 From: Isu Kim <49092508+isu-kim@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:46:53 +0900 Subject: [PATCH] Revert "Ml engine rework" --- .github/workflows/ci-test-py.yml | 45 ----------- ai-engine/.dockerignore | 6 -- ai-engine/.gitignore | 3 - ai-engine/Dockerfile | 22 ++---- ai-engine/Makefile | 6 +- ai-engine/ai-engine.py | 105 ++++++-------------------- ai-engine/client.py | 24 ------ ai-engine/requirements.txt | Bin 30 -> 0 bytes deployments/sentryflow.yaml | 37 --------- protobuf/Makefile | 2 +- protobuf/sentryflow_metrics.proto | 18 ----- sentryflow/config/config.go | 13 +--- sentryflow/metrics/api/apiAnalyzer.go | 3 - 13 files changed, 34 insertions(+), 250 deletions(-) delete mode 100644 .github/workflows/ci-test-py.yml delete mode 100644 ai-engine/.dockerignore delete mode 100644 ai-engine/.gitignore delete mode 100644 ai-engine/client.py delete mode 100644 ai-engine/requirements.txt delete mode 100644 protobuf/sentryflow_metrics.proto diff --git a/.github/workflows/ci-test-py.yml b/.github/workflows/ci-test-py.yml deleted file mode 100644 index ebe43d2..0000000 --- a/.github/workflows/ci-test-py.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: ci-test-py -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - py-pip-ai-sentryflow: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - cache: 'pip' - - - name: check Python pip3 - - run: pip install -r requirements.txt - - run: pip test - working-directory: ai-engine - - py-lint-ai-sentryflow: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - working-directory: ai-engine - - - name: Lint with Ruff - run: | - pip install ruff - ruff --output-format=github . - continue-on-error: true - working-directory: ai-engine diff --git a/ai-engine/.dockerignore b/ai-engine/.dockerignore deleted file mode 100644 index 23ca759..0000000 --- a/ai-engine/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -.idea -.git -.gitignore -protobuf -Dockerfile -__pycache__/ \ No newline at end of file diff --git a/ai-engine/.gitignore b/ai-engine/.gitignore deleted file mode 100644 index 533d889..0000000 --- a/ai-engine/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.idea/ -__pycache__/ -protobuf/ \ No newline at end of file diff --git a/ai-engine/Dockerfile b/ai-engine/Dockerfile index 1e40850..f2141b8 100644 --- a/ai-engine/Dockerfile +++ b/ai-engine/Dockerfile @@ -3,26 +3,14 @@ # Dockerfile FROM ubuntu:latest -RUN apt-get update && apt-get -y install python3 python3-pip wget git +RUN apt-get update && apt-get -y install python3 python3-pip wget git -RUN git clone https://github.com/isu-kim/stringlifier.git -WORKDIR ./stringlifier -RUN pip install . +RUN git clone https://github.com/adobe/stringlifier -RUN mkdir /app -WORKDIR /app -COPY /ai-engine . - -# Build protobuf for Python -RUN pip install grpcio grpcio-tools -RUN mkdir protobuf/ -COPY /protobuf ./protobuf - -# Due to python import bugs, we have to compile protoc using this command -# Refer to https://github.com/protocolbuffers/protobuf/issues/1491#issuecomment-261621112 for more information on this -RUN python3 -m grpc_tools.protoc --python_out=. --pyi_out=. --grpc_python_out=. -I=. protobuf/sentryflow_metrics.proto +RUN pip3 install ./stringlifier pymongo Flask WORKDIR /app -RUN pip install -r requirements.txt + +COPY . . CMD ["python3", "ai-engine.py"] diff --git a/ai-engine/Makefile b/ai-engine/Makefile index c42b031..9be3361 100644 --- a/ai-engine/Makefile +++ b/ai-engine/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 -IMAGE_NAME = 5gsec/sentryflow-ai-engine -TAG = v0.0.1 +IMAGE_NAME = 5GSEC/sentryflow-ai-engine +TAG = v0.1 .PHONY: build build: - docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile ../ + docker build -t $(IMAGE_NAME):$(TAG) -f ./Dockerfile diff --git a/ai-engine/ai-engine.py b/ai-engine/ai-engine.py index 8f1be34..9f025ce 100644 --- a/ai-engine/ai-engine.py +++ b/ai-engine/ai-engine.py @@ -1,94 +1,37 @@ -import os -import grpc - +from pymongo import MongoClient from stringlifier.api import Stringlifier -from concurrent import futures - -from protobuf import sentryflow_metrics_pb2_grpc -from protobuf import sentryflow_metrics_pb2 - - -class HandlerServer: - """ - Class for gRPC Servers - """ - def __init__(self): - try: - self.listen_addr = os.environ["AI_ENGINE_ADDRESS"] - except KeyError: - self.listen_addr = "0.0.0.0:5000" - - self.server = None - self.grpc_servers = list() - - def init_grpc_servers(self): - """ - init_grpc_servers method that initializes and registers gRPC servers - :return: None - """ - self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - self.grpc_servers.append(APIClassificationServer()) # @todo: make this configurable - - grpc_server: GRPCServer - for grpc_server in self.grpc_servers: - grpc_server.register(self.server) - - def serve(self): - """ - serve method that starts serving gRPC servers, this is blocking function. - :return: None - """ - self.server.add_insecure_port(self.listen_addr) - - print("[INFO] Starting to serve on {}".format(self.listen_addr)) - self.server.start() - self.server.wait_for_termination() +from flask import Flask +app = Flask(__name__) +s = Stringlifier() -class GRPCServer: - """ - Abstract class for an individual gRPC Server - """ - def register(self, server): - """ - register method that registers gRPC service to target server - :param server: The server - :return: None - """ - pass +@app.route('/api_metrics') +def api_metrics(): + # Connect to MongoDB + client = MongoClient('mongodb://mongo:27017') + # Access the numbat database + db = client.numbat -class APIClassificationServer(sentryflow_metrics_pb2_grpc.SentryFlowMetricsServicer, GRPCServer): - """ - Class for API Classification Server using Stringlifier - """ + # Access the access-logs collection + collection = db['access-logs'] - def __init__(self): - self.stringlifier = Stringlifier() - print("[Init] Successfully initialized APIClassificationServer") + # Retrieve all documents from the collection + logs = list(collection.find({})) - def register(self, server): - sentryflow_metrics_pb2_grpc.add_SentryFlowMetricsServicer_to_server(self, server) + # Close the MongoDB connection + client.close() - def GetAPIClassification(self, request_iterator, context): - """ - GetAPIClassification method that runs multiple API ML Classification at once - :param request_iterator: The requests - :param context: The context - :return: The results - """ + paths = list() - for req in request_iterator: - paths = req.paths - ml_results = self.stringlifier(paths) - print("{} -> {}".format(paths, ml_results)) + # Print out all entries + for log in logs: + paths.append(log["path"]) - results = [sentryflow_metrics_pb2.APIClassificationSingleResponse(merged=ml_result, fields=[]) for ml_result - in ml_results] - yield sentryflow_metrics_pb2.APIClassificationResponse(response=results) + parsed = s(paths) + print(set(parsed)) + return str(set(parsed)) if __name__ == '__main__': - hs = HandlerServer() - hs.init_grpc_servers() - hs.serve() + app.run(host='0.0.0.0', port=5000) diff --git a/ai-engine/client.py b/ai-engine/client.py deleted file mode 100644 index 8921127..0000000 --- a/ai-engine/client.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import uuid - -import grpc - -from protobuf import sentryflow_metrics_pb2_grpc -from protobuf import sentryflow_metrics_pb2 - -if __name__ == "__main__": - try: - listen_addr = os.environ["AI_ENGINE_ADDRESS"] - except KeyError: - listen_addr = "0.0.0.0:5000" - - with grpc.insecure_channel(listen_addr) as channel: - stub = sentryflow_metrics_pb2_grpc.SentryFlowMetricsStub(channel) - req = sentryflow_metrics_pb2.APIClassificationRequest(paths=["/api/test", "/api/test/" + str(uuid.uuid4())]) - - try: - response_stream = stub.GetAPIClassification(req) - for response in response_stream: - print("Response: ", str(response)) - except grpc.RpcError as e: - print("Error occurred during RPC:", e) \ No newline at end of file diff --git a/ai-engine/requirements.txt b/ai-engine/requirements.txt deleted file mode 100644 index 7c37043eaf4ff73168f413bd7dcb442e77a02844..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30 icmezWFP))?p@1QoA(J7W!4?P&8T1&;7z}`