From a2c932a490f7192d21a5f69990e70993e48cf9e9 Mon Sep 17 00:00:00 2001 From: Cezar Rata Date: Tue, 3 Sep 2024 15:28:54 -0700 Subject: [PATCH] Signed-off-by: Cezar Rata Signed-off-by: Cezar Rata Signed-off-by: Cezar Rata Signed-off-by: Cezar Rata Signed-off-by: Cezar Rata Signed-off-by: Cezar Rata porting patch #1 Signed-off-by: Cezar Rata --- api/handlers/image/image.go | 2 + api/handlers/image/load.go | 23 +++++++++ api/handlers/image/load_test.go | 70 ++++++++++++++++++++++++++ internal/backend/image.go | 16 ++++++ internal/service/image/load.go | 50 ++++++++++++++++++ internal/service/image/load_test.go | 62 +++++++++++++++++++++++ internal/service/image/tmp/tmp.txt | 0 mocks/mocks_backend/nerdctlimagesvc.go | 29 +++++++++++ mocks/mocks_image/imagesvc.go | 14 ++++++ setup-test-env.sh | 1 - 10 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 api/handlers/image/load.go create mode 100644 api/handlers/image/load_test.go create mode 100644 internal/service/image/load.go create mode 100644 internal/service/image/load_test.go create mode 100644 internal/service/image/tmp/tmp.txt diff --git a/api/handlers/image/image.go b/api/handlers/image/image.go index 42140ef..e9fb5af 100644 --- a/api/handlers/image/image.go +++ b/api/handlers/image/image.go @@ -24,6 +24,7 @@ type Service interface { Remove(ctx context.Context, name string, force bool) (deleted, untagged []string, err error) Tag(ctx context.Context, srcImg string, repo, tag string) error Inspect(ctx context.Context, name string) (*dockercompat.Image, error) + Load(ctx context.Context, inStream io.Reader, outStream io.Writer, quiet bool) error } func RegisterHandlers(r types.VersionedRouter, service Service, conf *config.Config, logger flog.Logger) { @@ -31,6 +32,7 @@ func RegisterHandlers(r types.VersionedRouter, service Service, conf *config.Con r.SetPrefix("/images") r.HandleFunc("/create", h.pull, http.MethodPost) + r.HandleFunc("/load", h.load, http.MethodPost) r.HandleFunc("/json", h.list, http.MethodGet) r.HandleFunc("/{name:.*}", h.remove, http.MethodDelete) r.HandleFunc("/{name:.*}/push", h.push, http.MethodPost) diff --git a/api/handlers/image/load.go b/api/handlers/image/load.go new file mode 100644 index 0000000..47afe2e --- /dev/null +++ b/api/handlers/image/load.go @@ -0,0 +1,23 @@ +package image + +import ( + "net/http" + "strconv" + + "github.com/containerd/containerd/namespaces" + "github.com/runfinch/finch-daemon/api/response" +) + +func (h *handler) load(w http.ResponseWriter, r *http.Request) { + ctx := namespaces.WithNamespace(r.Context(), h.Config.Namespace) + quiet, err := strconv.ParseBool(r.URL.Query().Get("quiet")) + if err != nil { + quiet = false + } + out := response.NewStreamWriter(w) + err = h.service.Load(ctx, r.Body, out, quiet) + if err != nil { + out.WriteError(http.StatusInternalServerError, err) + return + } +} diff --git a/api/handlers/image/load_test.go b/api/handlers/image/load_test.go new file mode 100644 index 0000000..f777920 --- /dev/null +++ b/api/handlers/image/load_test.go @@ -0,0 +1,70 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/containerd/nerdctl/pkg/config" + "github.com/golang/mock/gomock" + "github.com/gorilla/mux" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/runfinch/finch-daemon/mocks/mocks_image" + "github.com/runfinch/finch-daemon/mocks/mocks_logger" +) + +var _ = Describe("Image Load API", func() { + var ( + mockCtrl *gomock.Controller + logger *mocks_logger.Logger + service *mocks_image.MockService + h *handler + rr *httptest.ResponseRecorder + name string + req *http.Request + ) + BeforeEach(func() { + mockCtrl = gomock.NewController(GinkgoT()) + defer mockCtrl.Finish() + logger = mocks_logger.NewLogger(mockCtrl) + service = mocks_image.NewMockService(mockCtrl) + c := config.Config{} + h = newHandler(service, &c, logger) + rr = httptest.NewRecorder() + name = "test-image" + var err error + req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/images/%s/json", name), nil) + Expect(err).Should(BeNil()) + req = mux.SetURLVars(req, map[string]string{"name": name}) + + logger.EXPECT().Debugf(gomock.Any(), gomock.Any()).AnyTimes() + }) + Context("handler", func() { + It("should return 200 status code upon success", func() { + service.EXPECT().Load( + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Return(nil) + + // handler should return 200 status code + h.load(rr, req) + Expect(rr).Should(HaveHTTPStatus(http.StatusOK)) + }) + It("should return 500 status code if service returns an error message", func() { + service.EXPECT().Load(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("error")) + logger.EXPECT().Debugf(gomock.Any(), gomock.Any()) + + // handler should return error message + h.load(rr, req) + Expect(rr.Body).Should(MatchJSON(`{"message": "error"}`)) + Expect(rr).Should(HaveHTTPStatus(http.StatusInternalServerError)) + }) + }) +}) diff --git a/internal/backend/image.go b/internal/backend/image.go index 2730475..9f5ae1f 100644 --- a/internal/backend/image.go +++ b/internal/backend/image.go @@ -12,6 +12,8 @@ import ( "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" dockerconfig "github.com/containerd/containerd/remotes/docker/config" + "github.com/containerd/nerdctl/pkg/api/types" + "github.com/containerd/nerdctl/pkg/cmd/image" "github.com/containerd/nerdctl/pkg/idutil/imagewalker" "github.com/containerd/nerdctl/pkg/imageinspector" "github.com/containerd/nerdctl/pkg/imgutil" @@ -28,6 +30,8 @@ type NerdctlImageSvc interface { PullImage(ctx context.Context, stdout, stderr io.Writer, resolver remotes.Resolver, ref string, platforms []ocispec.Platform) (*imgutil.EnsuredImage, error) PushImage(ctx context.Context, resolver remotes.Resolver, tracker docker.StatusTracker, stdout io.Writer, pushRef, ref string, platMC platforms.MatchComparer) error SearchImage(ctx context.Context, name string) (int, int, []*images.Image, error) + LoadImage(ctx context.Context, img string, stdout io.Writer, quiet bool) error + GetDataStore() (string, error) } func (w *NerdctlWrapper) InspectImage(ctx context.Context, image images.Image) (*dockercompat.Image, error) { @@ -104,3 +108,15 @@ func (w *NerdctlWrapper) SearchImage(ctx context.Context, name string) (int, int n, err := walker.Walk(ctx, name) return n, uniqueCount, imgs, err } + +func (w *NerdctlWrapper) LoadImage(ctx context.Context, img string, stdout io.Writer, _ /*quiet*/ bool) error { + // TODO currently the "quiet" flag in nerdctl is hardcoded as "false". + // Ideally this flag should be part of the ImageLoadOptions, we can + // contribute this enhancement at upstream + return image.Load(ctx, w.clientWrapper.client, types.ImageLoadOptions{ + Stdout: stdout, + GOptions: *w.globalOptions, + Input: img, + AllPlatforms: true, + }) +} diff --git a/internal/service/image/load.go b/internal/service/image/load.go new file mode 100644 index 0000000..630bcd4 --- /dev/null +++ b/internal/service/image/load.go @@ -0,0 +1,50 @@ +package image + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/containerd/fifo" +) + +func (s *service) Load(ctx context.Context, inStream io.Reader, outStream io.Writer, quiet bool) error { + root, err := s.nctlImageSvc.GetDataStore() + if err != nil { + s.logger.Errorf("failed to get data store dir: %s", err) + return err + } + d := filepath.Join(root, "fifo") + if err = os.Mkdir(d, 0700); err != nil && !os.IsExist(err) { + fmt.Println(err) + s.logger.Errorf("failed to create fifo dir %s: %s", d, err) + return err + } + t := time.Now() + var b [3]byte + rand.Read(b[:]) + img := filepath.Join(d, fmt.Sprintf("%d%s", t.Nanosecond(), base64.URLEncoding.EncodeToString(b[:]))) + rw, err := fifo.OpenFifo(ctx, img, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700) + if err != nil { + s.logger.Errorf("failed to open fifo %s: %s", img, err) + return err + } + defer func() { + rw.Close() + os.Remove(img) + }() + go func() { + io.Copy(rw, inStream) + }() + if err = s.nctlImageSvc.LoadImage(ctx, img, outStream, quiet); err != nil { + s.logger.Errorf("failed to load image %s: %s", img, err) + return err + } + return nil +} diff --git a/internal/service/image/load_test.go b/internal/service/image/load_test.go new file mode 100644 index 0000000..9bbb7ac --- /dev/null +++ b/internal/service/image/load_test.go @@ -0,0 +1,62 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "context" + "errors" + "io" + "strings" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/runfinch/finch-daemon/api/handlers/image" + "github.com/runfinch/finch-daemon/mocks/mocks_backend" + "github.com/runfinch/finch-daemon/mocks/mocks_logger" +) + +// Unit tests related to image load API +var _ = Describe("Image Load API", func() { + var ( + ctx context.Context + mockCtrl *gomock.Controller + logger *mocks_logger.Logger + cdClient *mocks_backend.MockContainerdClient + ncClient *mocks_backend.MockNerdctlImageSvc + name string + inStream io.Reader + service image.Service + ) + BeforeEach(func() { + ctx = context.Background() + mockCtrl = gomock.NewController(GinkgoT()) + logger = mocks_logger.NewLogger(mockCtrl) + cdClient = mocks_backend.NewMockContainerdClient(mockCtrl) + ncClient = mocks_backend.NewMockNerdctlImageSvc(mockCtrl) + name = "tmp" + inStream = strings.NewReader("") + service = NewService(cdClient, ncClient, logger) + }) + Context("service", func() { + It("should return no errors upon success", func() { + // search image method returns one image + ncClient.EXPECT().GetDataStore().Return(name, nil) + ncClient.EXPECT().LoadImage(gomock.Any(), gomock.Any(), nil, gomock.Any()).Return(nil) + + // service should return no error + err := service.Load(ctx, inStream, nil, false) + Expect(err).Should(BeNil()) + }) + It("should return an error if no image was found", func() { + logger.EXPECT().Errorf(gomock.Any(), gomock.Any()) + ncClient.EXPECT().GetDataStore().Return("", errors.New("error message")) + + // service should return a error + err := service.Load(ctx, inStream, nil, false) + Expect(err).ShouldNot(BeNil()) + }) + }) +}) diff --git a/internal/service/image/tmp/tmp.txt b/internal/service/image/tmp/tmp.txt new file mode 100644 index 0000000..e69de29 diff --git a/mocks/mocks_backend/nerdctlimagesvc.go b/mocks/mocks_backend/nerdctlimagesvc.go index a911b26..c943655 100644 --- a/mocks/mocks_backend/nerdctlimagesvc.go +++ b/mocks/mocks_backend/nerdctlimagesvc.go @@ -43,6 +43,21 @@ func (m *MockNerdctlImageSvc) EXPECT() *MockNerdctlImageSvcMockRecorder { return m.recorder } +// GetDataStore mocks base method. +func (m *MockNerdctlImageSvc) GetDataStore() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDataStore") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDataStore indicates an expected call of GetDataStore. +func (mr *MockNerdctlImageSvcMockRecorder) GetDataStore() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataStore", reflect.TypeOf((*MockNerdctlImageSvc)(nil).GetDataStore)) +} + // GetDockerResolver mocks base method. func (m *MockNerdctlImageSvc) GetDockerResolver(arg0 context.Context, arg1 string, arg2 dockerconfigresolver.AuthCreds) (remotes.Resolver, docker.StatusTracker, error) { m.ctrl.T.Helper() @@ -74,6 +89,20 @@ func (mr *MockNerdctlImageSvcMockRecorder) InspectImage(arg0, arg1 interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InspectImage", reflect.TypeOf((*MockNerdctlImageSvc)(nil).InspectImage), arg0, arg1) } +// LoadImage mocks base method. +func (m *MockNerdctlImageSvc) LoadImage(arg0 context.Context, arg1 string, arg2 io.Writer, arg3 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadImage", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadImage indicates an expected call of LoadImage. +func (mr *MockNerdctlImageSvcMockRecorder) LoadImage(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadImage", reflect.TypeOf((*MockNerdctlImageSvc)(nil).LoadImage), arg0, arg1, arg2, arg3) +} + // PullImage mocks base method. func (m *MockNerdctlImageSvc) PullImage(arg0 context.Context, arg1, arg2 io.Writer, arg3 remotes.Resolver, arg4 string, arg5 []v1.Platform) (*imgutil.EnsuredImage, error) { m.ctrl.T.Helper() diff --git a/mocks/mocks_image/imagesvc.go b/mocks/mocks_image/imagesvc.go index 9fa3568..3be06c7 100644 --- a/mocks/mocks_image/imagesvc.go +++ b/mocks/mocks_image/imagesvc.go @@ -68,6 +68,20 @@ func (mr *MockServiceMockRecorder) List(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockService)(nil).List), arg0) } +// Load mocks base method. +func (m *MockService) Load(arg0 context.Context, arg1 io.Reader, arg2 io.Writer, arg3 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load. +func (mr *MockServiceMockRecorder) Load(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockService)(nil).Load), arg0, arg1, arg2, arg3) +} + // Pull mocks base method. func (m *MockService) Pull(arg0 context.Context, arg1, arg2, arg3 string, arg4 *types.AuthConfig, arg5 io.Writer) error { m.ctrl.T.Helper() diff --git a/setup-test-env.sh b/setup-test-env.sh index 3c0abde..8339845 100755 --- a/setup-test-env.sh +++ b/setup-test-env.sh @@ -41,4 +41,3 @@ sudo containerd & sudo buildkitd & sleep 2 -