From 04ed5a3dcefbca8eb415ca06a5c1868038eff527 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Thu, 25 Apr 2024 18:55:57 -0400 Subject: [PATCH] Opt to generate specs from structs --- client.yml | 109 -------------------------------- go.mod | 6 ++ go.sum | 15 ++++- internal/config/server.go | 56 ++++++++++++++++ main.go | 97 +++++----------------------- main_test.go | 74 +++++++++++----------- pkg/api/events.go | 74 ++++++++++++++++++++++ scyllaridae.complex-example.yml | 22 ------- scyllaridae.example.yml | 21 ++---- server.json | 65 ------------------- 10 files changed, 211 insertions(+), 328 deletions(-) delete mode 100644 client.yml create mode 100644 internal/config/server.go create mode 100644 pkg/api/events.go delete mode 100644 scyllaridae.complex-example.yml delete mode 100644 server.json diff --git a/client.yml b/client.yml deleted file mode 100644 index 0949f05..0000000 --- a/client.yml +++ /dev/null @@ -1,109 +0,0 @@ -openapi: 3.0.0 -info: - title: scyllaridae Service - description: Microservice to process events for Islandora. - version: "1.0.0" -servers: - - url: 'http://scyllaridae:8080' -paths: - /: - put: - summary: Runs a command based on the mimetype in the event - description: Receives media and other related information, performs processing based on MIME type and provided arguments and streams output to the destination location. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ProcessEvent' - responses: - '204': - description: Processing successful. - '400': - description: Bad request, possibly due to unsupported MIME type or malformed data. - '415': - description: Unsupported Media Type - '500': - description: Internal server error -components: - schemas: - ProcessEvent: - type: object - properties: - actor: - $ref: '#/components/schemas/Actor' - object: - $ref: '#/components/schemas/MediaObject' - attachment: - $ref: '#/components/schemas/Attachment' - type: - type: string - example: "Activity" - summary: - type: string - example: "Generate Derivative" - Actor: - type: object - properties: - id: - type: string - format: uri - MediaObject: - type: object - properties: - id: - type: string - format: uri - url: - type: array - items: - $ref: '#/components/schemas/Link' - isNewVersion: - type: boolean - Link: - type: object - properties: - name: - type: string - type: - type: string - enum: [Link] - href: - type: string - format: uri - mediaType: - type: string - rel: - type: string - Attachment: - type: object - properties: - type: - type: string - content: - $ref: '#/components/schemas/AttachmentContent' - mediaType: - type: string - AttachmentContent: - type: object - properties: - mimetype: - type: string - args: - type: string - source_uri: - type: string - format: uri - destination_uri: - type: string - format: uri - file_upload_uri: - type: string - format: uri - securitySchemes: - ApiKeyAuth: - type: apiKey - in: header - name: API-Key -security: - - ApiKeyAuth: [] diff --git a/go.mod b/go.mod index 7f5c53a..0882bef 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,9 @@ module github.com/lehigh-university-libraries/scyllaridae go 1.21.3 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/kr/pretty v0.3.1 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect +) diff --git a/go.sum b/go.sum index a62c313..67198fa 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,17 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/server.go b/internal/config/server.go new file mode 100644 index 0000000..91f0301 --- /dev/null +++ b/internal/config/server.go @@ -0,0 +1,56 @@ +package config + +// ServerConfig defines server-specific configurations. +// +// swagger:model ServerConfig +type ServerConfig struct { + // Label of the server configuration used for identification. + // + // required: true + Label string `yaml:"label"` + + // HTTP method used for sending data to the destination server. + // + // required: true + DestinationHTTPMethod string `yaml:"destinationHttpMethod"` + + // Header name for the file resource. + // + // required: false + FileHeader string `yaml:"fileHeader,omitempty"` + + // Header name for additional arguments passed to the command. + // + // required: false + ArgHeader string `yaml:"argHeader,omitempty"` + + // Indicates whether the authentication header should be forwarded. + // + // required: false + ForwardAuth bool `yaml:"forwardAuth,omitempty"` + + // List of MIME types allowed for processing. + // + // required: true + AllowedMimeTypes []string `yaml:"allowedMimeTypes"` + + // Commands and arguments ran by MIME type. + // + // required: true + CmdByMimeType map[string]Command `yaml:"cmdByMimeType"` +} + +// Command describes the command and arguments to execute for a specific MIME type. +// +// swagger:model Command +type Command struct { + // Command to execute. + // + // required: true + Cmd string `yaml:"cmd"` + + // Arguments for the command. + // + // required: false + Args []string `yaml:"args"` +} diff --git a/main.go b/main.go index e21f49d..8a5ebb5 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "encoding/json" "fmt" "log/slog" "net/http" @@ -10,67 +9,14 @@ import ( "os/exec" "strings" + scyllaridae "github.com/lehigh-university-libraries/scyllaridae/internal/config" + "github.com/lehigh-university-libraries/scyllaridae/pkg/api" + "gopkg.in/yaml.v3" ) -type Data struct { - Actor Actor `json:"actor"` - Object Object `json:"object"` - Attachment Attachment `json:"attachment"` - Type string `json:"type"` - Summary string `json:"summary"` -} - -type Actor struct { - Id string `json:"id"` -} - -type Object struct { - Id string `json:"id"` - URL []URL `json:"url"` - NewVersion bool `json:"isNewVersion"` -} - -type URL struct { - Name string `json:"name"` - Type string `json:"type"` - Href string `json:"href"` - MediaType string `json:"mediaType"` - Rel string `json:"rel"` -} - -type Attachment struct { - Type string `json:"type"` - Content Content `json:"content"` - MediaType string `json:"mediaType"` -} - -type Content struct { - MimeType string `json:"mimetype"` - Args string `json:"args"` - SourceUri string `json:"source_uri"` - DestinationUri string `json:"destination_uri"` - FileUploadUri string `json:"file_upload_uri"` - WebServiceUri string -} - -type Cmd struct { - Command string `yaml:"cmd,omitempty"` - Args []string `yaml:"args,omitempty"` -} - -type Config struct { - Label string `yaml:"label"` - Method string `yaml:"destination-http-method"` - FileHeader string `yaml:"file-header"` - ArgHeader string `yaml:"arg-header"` - ForwardAuth bool `yaml:"forward-auth"` - AllowedMimeTypes []string `yaml:"allowed-mimetypes"` - CmdByMimeType map[string]Cmd `yaml:"cmd-by-mimetype"` -} - var ( - config *Config + config *scyllaridae.ServerConfig ) func init() { @@ -105,7 +51,7 @@ func MessageHandler(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() // Read the Alpaca message payload - message, err := DecodeAlpacaMessage(r) + message, err := api.DecodeAlpacaMessage(r) if err != nil { slog.Error("Error decoding Pub/Sub message", "err", err) http.Error(w, "Error decoding Pub/Sub message", http.StatusInternalServerError) @@ -113,7 +59,7 @@ func MessageHandler(w http.ResponseWriter, r *http.Request) { } // Fetch the file contents from the URL - sourceResp, err := http.Get(message.Attachment.Content.SourceUri) + sourceResp, err := http.Get(message.Attachment.Content.SourceURI) if err != nil { slog.Error("Error fetching source file contents", "err", err) http.Error(w, "Error fetching file contents from URL", http.StatusInternalServerError) @@ -144,7 +90,7 @@ func MessageHandler(w http.ResponseWriter, r *http.Request) { } // Create the PUT request - req, err := http.NewRequest(config.Method, message.Attachment.Content.DestinationUri, &outBuf) + req, err := http.NewRequest(config.DestinationHTTPMethod, message.Attachment.Content.DestinationURI, &outBuf) if err != nil { slog.Error("Error creating HTTP request", "err", err) http.Error(w, "Error creating HTTP request", http.StatusInternalServerError) @@ -155,13 +101,13 @@ func MessageHandler(w http.ResponseWriter, r *http.Request) { req.Header.Set("Authorization", auth) } req.Header.Set("Content-Type", message.Attachment.Content.MimeType) - req.Header.Set("Content-Location", message.Attachment.Content.FileUploadUri) + req.Header.Set("Content-Location", message.Attachment.Content.FileUploadURI) // Execute the PUT request client := http.DefaultClient resp, err := client.Do(req) if err != nil { - slog.Error("Error sending request", "method", config.Method, "err", err) + slog.Error("Error sending request", "method", config.DestinationHTTPMethod, "err", err) http.Error(w, "Internal error.", http.StatusInternalServerError) return } @@ -169,7 +115,7 @@ func MessageHandler(w http.ResponseWriter, r *http.Request) { if resp.StatusCode > 299 { slog.Error("Request failed on destination server", "code", resp.StatusCode) - http.Error(w, fmt.Sprintf("%s request failed with status code %d", config.Method, resp.StatusCode), resp.StatusCode) + http.Error(w, fmt.Sprintf("%s request failed with status code %d", config.DestinationHTTPMethod, resp.StatusCode), resp.StatusCode) return } @@ -180,17 +126,7 @@ func MessageHandler(w http.ResponseWriter, r *http.Request) { } } -func DecodeAlpacaMessage(r *http.Request) (Data, error) { - var d Data - - if err := json.NewDecoder(r.Body).Decode(&d); err != nil { - return Data{}, err - } - - return d, nil -} - -func ReadConfig(yp string) (*Config, error) { +func ReadConfig(yp string) (*scyllaridae.ServerConfig, error) { var ( y []byte err error @@ -205,7 +141,7 @@ func ReadConfig(yp string) (*Config, error) { } } - var c Config + var c scyllaridae.ServerConfig err = yaml.Unmarshal(y, &c) if err != nil { return nil, err @@ -214,13 +150,12 @@ func ReadConfig(yp string) (*Config, error) { return &c, nil } -func buildExecCommand(mimetype, addtlArgs string, c *Config) (*exec.Cmd, error) { - var cmdConfig Cmd +func buildExecCommand(mimetype, addtlArgs string, c *scyllaridae.ServerConfig) (*exec.Cmd, error) { + var cmdConfig scyllaridae.Command var exists bool - slog.Info("Allowed formats", "formats", c.AllowedMimeTypes) if isAllowedMIMEType(mimetype, c.AllowedMimeTypes) { cmdConfig, exists = c.CmdByMimeType[mimetype] - if !exists || (len(cmdConfig.Command) == 0) { + if !exists || (len(cmdConfig.Cmd) == 0) { // Fallback to default if specific MIME type not configured or if command is empty cmdConfig = c.CmdByMimeType["default"] } @@ -237,7 +172,7 @@ func buildExecCommand(mimetype, addtlArgs string, c *Config) (*exec.Cmd, error) } } - cmd := exec.Command(cmdConfig.Command, args...) + cmd := exec.Command(cmdConfig.Cmd, args...) return cmd, nil } diff --git a/main_test.go b/main_test.go index f1dca98..dc50652 100644 --- a/main_test.go +++ b/main_test.go @@ -10,6 +10,8 @@ import ( "net/http/httptest" "os" "testing" + + "github.com/lehigh-university-libraries/scyllaridae/pkg/api" ) func TestMessageHandler_MethodNotAllowed(t *testing.T) { @@ -43,14 +45,14 @@ func TestIntegration_PutDestination(t *testing.T) { // Mock the environment variable for the configuration file path os.Setenv("SCYLLARIDAE_YML", ` -destination-http-method: "PUT" -file-header: Apix-Ldp-Resource -arg-header: X-Islandora-Args -forward-auth: false -allowed-mimetypes: [ +destinationHttpMethod: "PUT" +fileHeader: Apix-Ldp-Resource +argHeader: X-Islandora-Args +forwardAuth: false +allowedMimeTypes: [ "text/plain" ] -cmd-by-mimetype: +cmdByMimeType: default: cmd: "cat" `) @@ -66,13 +68,13 @@ cmd-by-mimetype: defer setupServer.Close() // Prepare a mock message to be sent to the main server - testData := Data{ - Actor: Actor{ - Id: "actor1", + testData := api.Payload{ + Actor: api.Actor{ + ID: "actor1", }, - Object: Object{ - Id: "object1", - URL: []URL{ + Object: api.Object{ + ID: "object1", + URL: []api.Link{ { Name: "Source", Type: "source", @@ -81,16 +83,16 @@ cmd-by-mimetype: Rel: "source", }, }, - NewVersion: true, + IsNewVersion: true, }, - Attachment: Attachment{ + Attachment: api.Attachment{ Type: "file", - Content: Content{ + Content: api.Content{ MimeType: "text/plain", Args: "", - SourceUri: sourceServer.URL, - DestinationUri: destinationServer.URL, - FileUploadUri: "", + SourceURI: sourceServer.URL, + DestinationURI: destinationServer.URL, + FileUploadURI: "", }, MediaType: "text/plain", }, @@ -135,14 +137,14 @@ func TestIntegration_GetDestination(t *testing.T) { // Mock the environment variable for the configuration file path os.Setenv("SCYLLARIDAE_YML", fmt.Sprintf(` -destination-http-method: "%s" -file-header: Apix-Ldp-Resource -arg-header: X-Islandora-Args -forward-auth: false -allowed-mimetypes: [ +destinationHttpMethod: "%s" +fileHeader: Apix-Ldp-Resource +argHeader: X-Islandora-Args +forwardAuth: false +allowedMimeTypes: [ "text/plain" ] -cmd-by-mimetype: +cmdByMimeType: default: cmd: "cat" `, method)) @@ -158,13 +160,13 @@ cmd-by-mimetype: defer setupServer.Close() // Prepare a mock message to be sent to the main server - testData := Data{ - Actor: Actor{ - Id: "actor1", + testData := api.Payload{ + Actor: api.Actor{ + ID: "actor1", }, - Object: Object{ - Id: "object1", - URL: []URL{ + Object: api.Object{ + ID: "object1", + URL: []api.Link{ { Name: "Source", Type: "source", @@ -173,16 +175,16 @@ cmd-by-mimetype: Rel: "source", }, }, - NewVersion: true, + IsNewVersion: true, }, - Attachment: Attachment{ + Attachment: api.Attachment{ Type: "file", - Content: Content{ + Content: api.Content{ MimeType: "text/plain", Args: "", - SourceUri: sourceServer.URL, - DestinationUri: destinationServer.URL, - FileUploadUri: "", + SourceURI: sourceServer.URL, + DestinationURI: destinationServer.URL, + FileUploadURI: "", }, MediaType: "text/plain", }, diff --git a/pkg/api/events.go b/pkg/api/events.go new file mode 100644 index 0000000..65086cb --- /dev/null +++ b/pkg/api/events.go @@ -0,0 +1,74 @@ +package api + +import ( + "encoding/json" + "net/http" +) + +// Payload defines the structure of the JSON payload received by the server. +// +// swagger:model Payload +type Payload struct { + Actor Actor `json:"actor" description:"Details of the actor performing the action"` + Object Object `json:"object" description:"Contains details about the object of the action"` + Attachment Attachment `json:"attachment" description:"Holds additional data related to the action"` + Type string `json:"type" description:"Type of the payload"` + Summary string `json:"summary" description:"Summary of the payload"` +} + +// Actor represents an entity performing an action. +// +// swagger:model Actor +type Actor struct { + ID string `json:"id" description:"Unique identifier for the actor"` +} + +// Object contains details about the object of the action. +// +// swagger:model Object +type Object struct { + ID string `json:"id" description:"Unique identifier for the object"` + URL []Link `json:"url" description:"List of hyperlinks related to the object"` + IsNewVersion bool `json:"isNewVersion" description:"Indicates if this is a new version of the object"` +} + +// Link describes a hyperlink related to the object. +// +// swagger:model Link +type Link struct { + Name string `json:"name" description:"Name of the link"` + Type string `json:"type" description:"Type of the link"` + Href string `json:"href" description:"Hyperlink reference URL"` + MediaType string `json:"mediaType" description:"Media type of the linked resource"` + Rel string `json:"rel" description:"Relationship type of the link"` +} + +// Attachment holds additional data related to the action. +// +// swagger:model Attachment +type Attachment struct { + Type string `json:"type" description:"Type of the attachment"` + Content Content `json:"content" description:"Content details within the attachment"` + MediaType string `json:"mediaType" description:"Media type of the attachment"` +} + +// Content describes specific content details in an attachment. +// +// swagger:model Content +type Content struct { + MimeType string `json:"mimetype" description:"MIME type of the content"` + Args string `json:"args" description:"Arguments used or applicable to the content"` + SourceURI string `json:"sourceUri" description:"Source URI from which the content is fetched"` + DestinationURI string `json:"destinationUri" description:"Destination URI to where the content is delivered"` + FileUploadURI string `json:"fileUploadUri" description:"File upload URI for uploading the content"` +} + +func DecodeAlpacaMessage(r *http.Request) (Payload, error) { + var p Payload + + if err := json.NewDecoder(r.Body).Decode(&p); err != nil { + return Payload{}, err + } + + return p, nil +} diff --git a/scyllaridae.complex-example.yml b/scyllaridae.complex-example.yml deleted file mode 100644 index fbe0e76..0000000 --- a/scyllaridae.complex-example.yml +++ /dev/null @@ -1,22 +0,0 @@ -label: "OCR PDFs and Images" -destination-http-method: "PUT" -file-header: Apix-Ldp-Resource -arg-header: X-Islandora-Args -forward-auth: false -allowed-mimetypes: [ - "application/pdf", - "image/*" -] -cmd-by-mimetype: - application/pdf: - cmd: "pdftotext" - args: - - "%s" - - "-" - - "-" - default: - cmd: "tesseract" - args: - - "stdin" - - "stdout" - - "%s" diff --git a/scyllaridae.example.yml b/scyllaridae.example.yml index 34e13a7..4ce516b 100644 --- a/scyllaridae.example.yml +++ b/scyllaridae.example.yml @@ -1,17 +1,10 @@ -# take the input from a source server -# and print it to the output server -# -# Though since we're using destination-http-method: "GET" -# the destination server isn't receiving anything -# this example can be used to warm the cache with a properly formatted event -label: "Warm cache" -destination-http-method: "GET" -file-header: Apix-Ldp-Resource -arg-header: X-Islandora-Args -forward-auth: false -allowed-mimetypes: [ - "text/plain" +destinationHttpMethod: GET +fileHeader: Apix-Ldp-Resource +argHeader: X-Islandora-Args +forwardAuth: false +allowedMimeTypes: [ + "text/html" ] -cmd-by-mimetype: +cmdByMimeType: default: cmd: "cat" diff --git a/server.json b/server.json deleted file mode 100644 index 1b2657c..0000000 --- a/server.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Server Configuration for scyllaridae", - "type": "object", - "properties": { - "label": { - "type": "string", - "description": "Label of the server configuration. Only used to help identify what the config does." - }, - "destination-http-method": { - "type": "string", - "description": "HTTP method used for sending data to the destination server." - }, - "file-header": { - "type": "string", - "description": "Header name for the file resource if it's not passed in the event." - }, - "arg-header": { - "type": "string", - "description": "Header name for additional arguments passed to the command." - }, - "forward-auth": { - "type": "boolean", - "description": "Indicates whether the authentication header should be forwarded." - }, - "allowed-mimetypes": { - "type": "array", - "items": { - "type": "string" - }, - "description": "List of MIME types allowed for processing. Either the full mimetype or globs image/*" - }, - "cmd-by-mimetype": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "cmd": { - "type": "string", - "description": "Command to execute." - }, - "args": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Arguments for the command. %s is a special value used to place the value from arg-header" - } - }, - "required": ["cmd", "args"] - }, - "description": "Commands and arguments ran by MIME type. The default mimetype is used for all mimetypes if an allowed mimetype does not have an explicit command." - } - }, - "required": [ - "label", - "destination-http-method", - "file-header", - "arg-header", - "forward-auth", - "allowed-mimetypes", - "cmd-by-mimetype" - ] -} -