Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[minor] Support POST a file to a service instead of referencing a URL #47

Merged
merged 2 commits into from
Oct 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/lint-test-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: lint-test-build-push
on:
push:
paths-ignore:
- '*.md'
- '**/*.md'
- '.github/**'
- 'ci/**'

Expand Down
27 changes: 26 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,31 @@
# scyllaridae

Any command that takes stdin as input and streams its output to stdout can use scyllaridae.
Any command that takes a file as input and prints a result as output can use scyllaridae.

Both `GET` and `POST` requests are supported.

`GET` supports Islandora's alpaca/event spec, which sends a URL of a file as an HTTP header `Apix-Ldp-Resource` and prints the result. e.g. to create a VTT file from an audio file:

```
curl -H "Apix-Ldp-Resource: https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav" \
http://localhost:8080
WEBVTT

00:00:00.000 --> 00:00:11.000
And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
```

`POST` supports directly uploading a file to the service

```
curl -H "Content-Type: audio/x-wav" \
--data-binary "@output.wav" \
http://localhost:8080/
WEBVTT

00:00:00.000 --> 00:00:02.960
Lehigh University Library Technology.
```

## Adding a new microservice

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/go-stomp/stomp/v3 v3.1.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/stretchr/testify v1.9.0
golang.org/x/text v0.3.3
gopkg.in/yaml.v3 v3.0.1
)

Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down
28 changes: 28 additions & 0 deletions internal/config/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ package config

import (
"fmt"
"io"
"log/slog"
"mime"
"net/http"
"os"
"os/exec"
"regexp"
Expand Down Expand Up @@ -279,3 +282,28 @@ func GetPassedArgs(args string) ([]string, error) {

return passedArgs, nil
}

func (c *ServerConfig) GetFileStream(r *http.Request, message api.Payload, auth string) (io.ReadCloser, int, error) {
if r.Method == http.MethodPost {
return r.Body, http.StatusOK, nil
}
req, err := http.NewRequest("GET", message.Attachment.Content.SourceURI, nil)
if err != nil {
slog.Error("Error building request to fetch source file contents", "err", err)
return nil, http.StatusBadRequest, fmt.Errorf("bad request")
}
if c.ForwardAuth {
req.Header.Set("Authorization", auth)
}
sourceResp, err := http.DefaultClient.Do(req)
if err != nil {
slog.Error("Error fetching source file contents", "err", err)
return nil, http.StatusInternalServerError, fmt.Errorf("internal error")
}
if sourceResp.StatusCode != http.StatusOK {
slog.Error("SourceURI sent a bad status code", "code", sourceResp.StatusCode, "uri", message.Attachment.Content.SourceURI)
return nil, http.StatusFailedDependency, fmt.Errorf("failed dependency")
}

return sourceResp.Body, http.StatusOK, nil
}
6 changes: 3 additions & 3 deletions main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestMessageHandler_MethodNotAllowed(t *testing.T) {
testConfig := &scyllaridae.ServerConfig{}
server := &Server{Config: testConfig}

req, err := http.NewRequest("POST", "/", nil)
req, err := http.NewRequest("PUT", "/", nil)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -69,9 +69,9 @@ cmdByMimeType:
name: "cURL fail with bad auth",
authHeader: "foo",
requestAuth: "bar",
expectedStatus: http.StatusBadRequest,
expectedStatus: http.StatusFailedDependency,
returnedBody: "foo",
expectedBody: "Bad request\n",
expectedBody: "Failed Dependency\n",
expectMismatch: false,
mimetype: "text/plain",
destinationMimeType: "application/xml",
Expand Down
22 changes: 12 additions & 10 deletions pkg/api/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,18 @@ func DecodeEventMessage(msg []byte) (Payload, error) {
func DecodeAlpacaMessage(r *http.Request, auth string) (Payload, error) {
p := Payload{}

p.Attachment.Content.Args = r.Header.Get("X-Islandora-Args")
p.Attachment.Content.SourceURI = r.Header.Get("Apix-Ldp-Resource")
p.Attachment.Content.DestinationMimeType = r.Header.Get("Accept")
p.Attachment.Content.SourceMimeType = r.Header.Get("Content-Type")
if p.Attachment.Content.DestinationMimeType == "" {
p.Attachment.Content.DestinationMimeType = "text/plain"
}

if r.Method == http.MethodPost {
return p, nil
}

// if the message was sent in the event header, just read it
message := r.Header.Get("X-Islandora-Event")
if message != "" {
Expand All @@ -92,21 +104,11 @@ func DecodeAlpacaMessage(r *http.Request, auth string) (Payload, error) {
slog.Error("Error decoding base64", "err", err)
return p, err
}
slog.Info("Received message", "msg", j)
err = json.Unmarshal(j, &p)
if err != nil {
slog.Error("Error unmarshalling event", "err", err)
return p, err
}
// else if this is a standard alpaca request, get the event from the headers alpaca sends
} else {
p.Attachment.Content.Args = r.Header.Get("X-Islandora-Args")
p.Attachment.Content.SourceURI = r.Header.Get("Apix-Ldp-Resource")

p.Attachment.Content.DestinationMimeType = r.Header.Get("Accept")
if p.Attachment.Content.DestinationMimeType == "" {
p.Attachment.Content.DestinationMimeType = "text/plain"
}
}

err := p.getSourceUri(auth)
Expand Down
35 changes: 11 additions & 24 deletions server.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (

scyllaridae "github.com/lehigh-university-libraries/scyllaridae/internal/config"
"github.com/lehigh-university-libraries/scyllaridae/pkg/api"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)

type Server struct {
Expand Down Expand Up @@ -38,13 +40,13 @@ func runHTTPServer(server *Server) {
func (s *Server) MessageHandler(w http.ResponseWriter, r *http.Request) {
slog.Info(r.RequestURI, "method", r.Method, "ip", r.RemoteAddr, "proto", r.Proto)

if r.Method != http.MethodGet {
if r.Method != http.MethodGet && r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
defer r.Body.Close()

if r.Header.Get("Apix-Ldp-Resource") == "" && r.Header.Get("X-Islandora-Event") == "" {
if r.Header.Get("Apix-Ldp-Resource") == "" && r.Header.Get("X-Islandora-Event") == "" && r.Method == http.MethodGet {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
Expand All @@ -61,36 +63,21 @@ func (s *Server) MessageHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Stream the file contents from the source URL
req, err := http.NewRequest("GET", message.Attachment.Content.SourceURI, nil)
if err != nil {
slog.Error("Error creating request to source", "source", message.Attachment.Content.SourceURI, "err", err)
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
if s.Config.ForwardAuth {
req.Header.Set("Authorization", auth)
}
sourceResp, err := http.DefaultClient.Do(req)
cmd, err := scyllaridae.BuildExecCommand(message, s.Config)
if err != nil {
slog.Error("Error fetching source file contents", "err", err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
defer sourceResp.Body.Close()
if sourceResp.StatusCode != http.StatusOK {
slog.Error("SourceURI sent a bad status code", "code", sourceResp.StatusCode, "uri", message.Attachment.Content.SourceURI)
slog.Error("Error building command", "err", err)
http.Error(w, "Bad request", http.StatusBadRequest)
return
}

cmd, err := scyllaridae.BuildExecCommand(message, s.Config)
// Stream the file contents from the source URL or request body
fs, errCode, err := s.Config.GetFileStream(r, message, auth)
if err != nil {
slog.Error("Error building command", "err", err)
http.Error(w, "Bad request", http.StatusBadRequest)
http.Error(w, cases.Title(language.English).String(fmt.Sprint(err)), errCode)
return
}
cmd.Stdin = sourceResp.Body
defer fs.Close()
cmd.Stdin = fs

// Create a buffer to capture stderr
var stdErr bytes.Buffer
Expand Down
Loading