Merge pull request #1 from RunbookSolutions/staging

Staging

.github/workflows/production.yml

@@ -0,0 +1,34 @@
+name: Production
+
+on:
+  push:
+    branches:
+      - 'production'
+  release:
+    types:
+      - created
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: |
+            runbooksolutions/agent:latest
+            ${{ github.event_name == 'release' && github.ref ? 'runbooksolutions/agent:' + github.ref : '' }}

.github/workflows/staging.yml

@@ -0,0 +1,29 @@
+name: Dev
+
+on:
+  push:
+    branches:
+      - 'staging'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: runbooksolutions/agent:dev

.gitignore

@@ -0,0 +1,6 @@
+kerberos
+run
+OLD
+hosts
+test.py
+__pycache__

Dockerfile

@@ -0,0 +1,26 @@
+# Use an official Python runtime as a parent image
+FROM python:latest
+
+# Install Required System Tools
+RUN apt-get update && \
+    apt-get install nmap gcc libkrb5-dev libssl-dev krb5-user -y
+
+# Set the working directory to /app
+WORKDIR /app
+
+RUN mkdir plugins,stores
+
+COPY _docker/start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Copy the current directory contents into the container at /app
+COPY requirements.txt /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install -r requirements.txt
+
+COPY app.py /app
+COPY runbooksolutions /app/runbooksolutions
+
+# Define the command to run your application
+CMD [ "/start.sh" ]

README.md

@@ -1 +1,83 @@
 # RunbookSolution Network Agent
+
+This codebase comprises the core of the Network Agent for RunbookSolutions.
+
+## Installation
+
+### Prebuilt Docker Image
+
+```sh
+mkdir agent
+cd agent
+mkdir plugins stores kerberos
+wget https://raw.githubusercontent.com/RunbookSolutions/agent/staging/config.ini
+
+docker run \
+    --name RunbookSolutions_Agent \
+    -v $(pwd)/config.ini:/app/config.ini \
+    -v $(pwd)/plugins:/app/plugins \
+    -v $(pwd)/stores:/app/stores \
+    -v $(pwd)/kerberos/krb5.conf:/etc/krb5.conf \
+    -v $(pwd)/kerberos:/keytabs \
+    -d \
+    --restart unless-stopped \
+    runbooksolutions/agent:latest
+
+```
+
+### Extending the Default Image
+
+The default image includes the following Python libraries by default. To include additional libraries for your custom plugins, simply extend our default image and use it instead.
+
+```Dockerfile
+FROM runbooksolutions/agent:latest
+# Using a requirements.txt (Recommended)
+COPY requirements.txt /app/custom_requirements.txt
+RUN pip install -r custom_requirements.txt
+# OR Individually
+RUN pip install some_package
+```
+
+
+### From Source
+```sh
+git clone https://github.com/RunbookSolutions/agent.git
+cd agent
+./run
+```
+
+## Configuration
+Configuration is maintained in a simple 'config.ini' file consisting of the 'server_url' of the backend and the 'client_id' for device authentication.
+
+```ini
+[agent]
+server_url=http://192.168.1.197 # Note: Do NOT include a trailing slash on the server_url
+client_id=9ab55261-bfb7-4bb3-ad29-a6dbdbf8a5af # Device Code Grant client_id provided by the server
+auth=True # To disable auth when not using with RunbookSolutions.
+```
+
+## Expected Server Responses
+Due to the agent's nature, it can easily be used by others outside of RunbookSolutions.
+To implement a backend for this agent, you will need to provide the following endpoints.
+
+`GET /api/agent` for the agent to load information about itself. This endpoint also provides the agent with a list of PLUGIN_IDs that it needs to load.
+
+`GET /api/agent/plugin/{PLUGIN_ID}` for the agent to download plugins. This endpoint also provides details about commands the plugin provides.
+
+`GET /api/agent/tasks` for the agent to load tasks that it needs to run. Tasks include scheduled and one-off tasks to run and will always present tasks until they are removed from the backend. This allows the agent to restart without skipping task execution.
+
+Additional details can be found on the [Expected Server Responses](/docs/Responses.md) page.
+
+## Creating a Keytab File
+
+Some plugins may require authentication against your windows domain.
+
+The simplest way to acomplish this is by using the [Docker Kerberos Keytab Generator](https://github.com/simplesteph/docker-kerberos-get-keytab):
+
+```sh
+cd agent
+docker run -it --rm \
+            -v $(pwd)/kerberos:/output \
+            -e PRINCIPAL=<[email protected]> \
+            simplesteph/docker-kerberos-get-keytab
+```

_docker/start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+keytab_directory="/keytabs"
+
+for keytab_file in "$keytab_directory"/*.keytab; do
+    identity=$(basename "$keytab_file" .keytab)
+    kinit -kt "$keytab_file" "$identity"
+done
+
+python app.py

app.py

@@ -0,0 +1,22 @@
+
+import logging
+from runbooksolutions.logging_setup import setup_logging
+setup_logging()
+
+
+from runbooksolutions.store.Store import Store
+# Set the Data Store(s) Password
+Store.set_encryption_key(b"Store_Encryption_Password")
+from runbooksolutions.agent.Agent import Agent
+
+async def main():
+    agent = Agent(num_threads=3)
+    try:
+        await agent.start()
+    except KeyboardInterrupt:
+        logging.info("Received CTRL+C. Stopping gracefully.")
+        await agent.stop()
+
+import asyncio
+if __name__ == "__main__":
+    asyncio.run(main())

config.ini

@@ -0,0 +1,7 @@
+[agent]
+# Note: Do NOT include a trailing slash on the server_url
+server_url=http://192.168.1.197
+# Device Code Grant client_id provided by the server
+client_id=9ab55261-bfb7-4bb3-ad29-a6dbdbf8a5af
+# If we are required to preform Device Code Authentication
+auth=True

docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3'
+services:
+  network-agent:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - .:/app
+    environment:
+      - DEBUG=true

docs/Responses.md

@@ -0,0 +1,88 @@
+# Responses From Backend
+
+> Note: The backend server is expected to identify the agent making the request without any additional parameters being sent. By default RunbookSolutions achieves this using the OAuth Device Code process to retrieve an Access Token for authentication.
+>> In non-NAT'ed environments; the backend server could use the IP address from the request to identify agents.
+
+## `GET /api/agent`
+
+This endpoint provides the agent with basic information about itself along with a list of PLUGIN_ID's that the agent should have loaded.
+
+- The Team ID provided here is used to identify and group agents to specific groups on the backend. It is **required** even if not used.
+
+
+```json
+{
+	"data": {
+		"id": "9ab55db0-9bfa-45a6-8280-bb94c6b0fe8d",
+		"name": "Test Agent",
+		"team_id": "9ab55261-b6b0-4fb4-85e5-a3491a72f720",
+		"plugins": [
+			"9ab56426-f429-4c4b-9755-40c92449f0be"
+		]
+	}
+}
+```
+
+## `GET /api/agent/plugin/{plugin_id}`
+This endpoint provides the agent with individual plugins for the agent along with the corresponding commands the plugin makes available.
+
+> Note: It is important to note that two different versions of a plugin may be loaded by an agent. Commands are prefixed with the PLUGIN_ID to avoid collisions.
+
+- The `script` variable contains the code the agent will execute when required.
+- The `hash` variable is the `SHA512` hash of the script; the agent will verify both the script variable as well as the file it creates to store the plugin.
+- The `commands` variable contains the details of what function in the program to run for which command is provided.
+
+> Important: Both the `script` variable and the file written to disk must match the provided hash for the plugin to be loaded and run.
+
+```json
+{
+	"data": {
+		"id": "9ab56426-f429-4c4b-9755-40c92449f0be",
+		"name": "Test Plugin",
+		"version": 0,
+		"description": null,
+		"script": "class Plugin:\n    def __init__(self):\n        pass\n    def greet(self):\n        print(\"Hello from the Test Plugin!\")\n    def square(self, number):\n        result = number ** 2\n        print(f\"The square of {number} is {result}\")",
+		"hash": "809c167b7b1e7fb9504dc136af3c2dc1c17545355a9aaec28c3792e54bc540943db236b6af547a732161b5d717c9b14a7c508ab49b3f06e128997de06b3abfd3",
+		"commands": {
+			"9ab56426-f429-4c4b-9755-40c92449f0be.greet": {
+				"id": "9ab5659c-271d-40d5-be37-3a3847b92aab",
+				"name": "9ab56426-f429-4c4b-9755-40c92449f0be.greet",
+				"function": "greet"
+			},
+			"9ab56426-f429-4c4b-9755-40c92449f0be.square": {
+				"id": "9ab565fc-1036-4afb-ac7e-ec92e0db6985",
+				"name": "9ab56426-f429-4c4b-9755-40c92449f0be.square",
+				"function": "square"
+			}
+		}
+	}
+}
+```
+
+## `GET /api/agent/tasks`
+The following endpoint provides the agent with details of what commands need to be run, when (if scheduled), and any arguments for said command.
+
+- The `command` variable must match one of the keys provided by the `plugin.commands` variable when downloading plugins.
+- The `cron` variable is the cron formatted schedule for when the task runs, or `null` if it should only be run once.
+- The `arguments` variable should be a JSON encoded string containing the argument name and values for the function to run.
+
+> Note: The agent uses the `task.id` to ensure tasks are not being duplicated into the schedule and queue.
+
+```json
+{
+	"data": [
+		{
+			"id": "9ab581cf-546d-405a-afaf-474cc631ed5c",
+			"command": "9ab56426-f429-4c4b-9755-40c92449f0be.greet",
+			"cron": null,
+			"arguments": "{}"
+		},
+		{
+			"id": "9ab582f7-9e64-4fad-b6b9-369633776ae4",
+			"command": "9ab56426-f429-4c4b-9755-40c92449f0be.square",
+			"cron": "* * * * *",
+			"arguments": "{\"number\":2}"
+		}
+	]
+}
+```

plugins/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

requirements.txt

@@ -0,0 +1,4 @@
+requests
+cryptography
+colorlog
+croniter

run

@@ -0,0 +1,7 @@
+docker run \
+    -v ./config.ini:/app/config.ini \
+    -v ./plugins:/app/plugins \
+    -v ./stores:/app/stores \
+    -v $(pwd)/kerberos/krb5.conf:/etc/krb5.conf \
+    -v $(pwd)/kerberos:/keytabs \
+    -it $(docker build -q .)