From c5462259b09b4e6829fefcdf5b26d926e742e0d2 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 12 Oct 2024 13:58:28 -0400 Subject: [PATCH] [minor] Support POST a file to a service instead of referencing a URL (#47) --- .github/workflows/lint-test-build.yml | 2 +- README.md | 27 ++++++++++++++++++++- go.mod | 1 + go.sum | 1 + internal/config/server.go | 28 +++++++++++++++++++++ main_test.go | 6 ++--- pkg/api/events.go | 22 +++++++++-------- server.go | 35 +++++++++------------------ 8 files changed, 83 insertions(+), 39 deletions(-) diff --git a/.github/workflows/lint-test-build.yml b/.github/workflows/lint-test-build.yml index 12753d3..634266e 100644 --- a/.github/workflows/lint-test-build.yml +++ b/.github/workflows/lint-test-build.yml @@ -2,7 +2,7 @@ name: lint-test-build-push on: push: paths-ignore: - - '*.md' + - '**/*.md' - '.github/**' - 'ci/**' diff --git a/README.md b/README.md index 4aefa80..dbad40b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,31 @@ # scyllaridae -Any command that takes stdin as input and streams its output to stdout can use scyllaridae. +Any command that takes a file as input and prints a result as output can use scyllaridae. + +Both `GET` and `POST` requests are supported. + +`GET` supports Islandora's alpaca/event spec, which sends a URL of a file as an HTTP header `Apix-Ldp-Resource` and prints the result. e.g. to create a VTT file from an audio file: + +``` +curl -H "Apix-Ldp-Resource: https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav" \ + http://localhost:8080 +WEBVTT + +00:00:00.000 --> 00:00:11.000 + And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country. +``` + +`POST` supports directly uploading a file to the service + +``` +curl -H "Content-Type: audio/x-wav" \ + --data-binary "@output.wav" \ + http://localhost:8080/ +WEBVTT + +00:00:00.000 --> 00:00:02.960 + Lehigh University Library Technology. +``` ## Adding a new microservice diff --git a/go.mod b/go.mod index ae234c3..db2ae32 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-stomp/stomp/v3 v3.1.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/stretchr/testify v1.9.0 + golang.org/x/text v0.3.3 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 8911a53..a836368 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/internal/config/server.go b/internal/config/server.go index 52f0135..84d042f 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -2,7 +2,10 @@ package config import ( "fmt" + "io" + "log/slog" "mime" + "net/http" "os" "os/exec" "regexp" @@ -279,3 +282,28 @@ func GetPassedArgs(args string) ([]string, error) { return passedArgs, nil } + +func (c *ServerConfig) GetFileStream(r *http.Request, message api.Payload, auth string) (io.ReadCloser, int, error) { + if r.Method == http.MethodPost { + return r.Body, http.StatusOK, nil + } + req, err := http.NewRequest("GET", message.Attachment.Content.SourceURI, nil) + if err != nil { + slog.Error("Error building request to fetch source file contents", "err", err) + return nil, http.StatusBadRequest, fmt.Errorf("bad request") + } + if c.ForwardAuth { + req.Header.Set("Authorization", auth) + } + sourceResp, err := http.DefaultClient.Do(req) + if err != nil { + slog.Error("Error fetching source file contents", "err", err) + return nil, http.StatusInternalServerError, fmt.Errorf("internal error") + } + if sourceResp.StatusCode != http.StatusOK { + slog.Error("SourceURI sent a bad status code", "code", sourceResp.StatusCode, "uri", message.Attachment.Content.SourceURI) + return nil, http.StatusFailedDependency, fmt.Errorf("failed dependency") + } + + return sourceResp.Body, http.StatusOK, nil +} diff --git a/main_test.go b/main_test.go index 327a682..721095f 100644 --- a/main_test.go +++ b/main_test.go @@ -28,7 +28,7 @@ func TestMessageHandler_MethodNotAllowed(t *testing.T) { testConfig := &scyllaridae.ServerConfig{} server := &Server{Config: testConfig} - req, err := http.NewRequest("POST", "/", nil) + req, err := http.NewRequest("PUT", "/", nil) if err != nil { t.Fatal(err) } @@ -69,9 +69,9 @@ cmdByMimeType: name: "cURL fail with bad auth", authHeader: "foo", requestAuth: "bar", - expectedStatus: http.StatusBadRequest, + expectedStatus: http.StatusFailedDependency, returnedBody: "foo", - expectedBody: "Bad request\n", + expectedBody: "Failed Dependency\n", expectMismatch: false, mimetype: "text/plain", destinationMimeType: "application/xml", diff --git a/pkg/api/events.go b/pkg/api/events.go index 73fcee5..522738a 100644 --- a/pkg/api/events.go +++ b/pkg/api/events.go @@ -84,6 +84,18 @@ func DecodeEventMessage(msg []byte) (Payload, error) { func DecodeAlpacaMessage(r *http.Request, auth string) (Payload, error) { p := Payload{} + p.Attachment.Content.Args = r.Header.Get("X-Islandora-Args") + p.Attachment.Content.SourceURI = r.Header.Get("Apix-Ldp-Resource") + p.Attachment.Content.DestinationMimeType = r.Header.Get("Accept") + p.Attachment.Content.SourceMimeType = r.Header.Get("Content-Type") + if p.Attachment.Content.DestinationMimeType == "" { + p.Attachment.Content.DestinationMimeType = "text/plain" + } + + if r.Method == http.MethodPost { + return p, nil + } + // if the message was sent in the event header, just read it message := r.Header.Get("X-Islandora-Event") if message != "" { @@ -92,21 +104,11 @@ func DecodeAlpacaMessage(r *http.Request, auth string) (Payload, error) { slog.Error("Error decoding base64", "err", err) return p, err } - slog.Info("Received message", "msg", j) err = json.Unmarshal(j, &p) if err != nil { slog.Error("Error unmarshalling event", "err", err) return p, err } - // else if this is a standard alpaca request, get the event from the headers alpaca sends - } else { - p.Attachment.Content.Args = r.Header.Get("X-Islandora-Args") - p.Attachment.Content.SourceURI = r.Header.Get("Apix-Ldp-Resource") - - p.Attachment.Content.DestinationMimeType = r.Header.Get("Accept") - if p.Attachment.Content.DestinationMimeType == "" { - p.Attachment.Content.DestinationMimeType = "text/plain" - } } err := p.getSourceUri(auth) diff --git a/server.go b/server.go index 9bd25a2..651372d 100644 --- a/server.go +++ b/server.go @@ -9,6 +9,8 @@ import ( scyllaridae "github.com/lehigh-university-libraries/scyllaridae/internal/config" "github.com/lehigh-university-libraries/scyllaridae/pkg/api" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) type Server struct { @@ -38,13 +40,13 @@ func runHTTPServer(server *Server) { func (s *Server) MessageHandler(w http.ResponseWriter, r *http.Request) { slog.Info(r.RequestURI, "method", r.Method, "ip", r.RemoteAddr, "proto", r.Proto) - if r.Method != http.MethodGet { + if r.Method != http.MethodGet && r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) return } defer r.Body.Close() - if r.Header.Get("Apix-Ldp-Resource") == "" && r.Header.Get("X-Islandora-Event") == "" { + if r.Header.Get("Apix-Ldp-Resource") == "" && r.Header.Get("X-Islandora-Event") == "" && r.Method == http.MethodGet { http.Error(w, "Bad request", http.StatusBadRequest) return } @@ -61,36 +63,21 @@ func (s *Server) MessageHandler(w http.ResponseWriter, r *http.Request) { return } - // Stream the file contents from the source URL - req, err := http.NewRequest("GET", message.Attachment.Content.SourceURI, nil) - if err != nil { - slog.Error("Error creating request to source", "source", message.Attachment.Content.SourceURI, "err", err) - http.Error(w, "Bad request", http.StatusBadRequest) - return - } - if s.Config.ForwardAuth { - req.Header.Set("Authorization", auth) - } - sourceResp, err := http.DefaultClient.Do(req) + cmd, err := scyllaridae.BuildExecCommand(message, s.Config) if err != nil { - slog.Error("Error fetching source file contents", "err", err) - http.Error(w, "Internal error", http.StatusInternalServerError) - return - } - defer sourceResp.Body.Close() - if sourceResp.StatusCode != http.StatusOK { - slog.Error("SourceURI sent a bad status code", "code", sourceResp.StatusCode, "uri", message.Attachment.Content.SourceURI) + slog.Error("Error building command", "err", err) http.Error(w, "Bad request", http.StatusBadRequest) return } - cmd, err := scyllaridae.BuildExecCommand(message, s.Config) + // Stream the file contents from the source URL or request body + fs, errCode, err := s.Config.GetFileStream(r, message, auth) if err != nil { - slog.Error("Error building command", "err", err) - http.Error(w, "Bad request", http.StatusBadRequest) + http.Error(w, cases.Title(language.English).String(fmt.Sprint(err)), errCode) return } - cmd.Stdin = sourceResp.Body + defer fs.Close() + cmd.Stdin = fs // Create a buffer to capture stderr var stdErr bytes.Buffer