diff --git a/core/config/config.go b/core/config/config.go index 79072798..92454dcd 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -27,6 +27,7 @@ type Runtime struct { Executable string `toml:"executable" json:"executable,omitempty"` // relative path to the binary Cmd []string `toml:"cmd" json:"cmd,omitempty"` Paths []string `toml:"paths" json:"paths,omitempty"` // (optional) extra directories added to PATH variable, MUST be relative + ReadOnly bool `toml:"read_only" json:"read_only,omitempty"` } func DefaultConfig[R any]() *Config[R] { diff --git a/core/mock/gen.go b/core/mock/gen.go index 93ce3568..5a7a3023 100644 --- a/core/mock/gen.go +++ b/core/mock/gen.go @@ -20,4 +20,5 @@ //go:generate mockgen -typed -destination=store/oci/manager.go -source=../pkg/store/oci/manager.go //go:generate mockgen -typed -destination=runtime/provider.go -source=../pkg/runtime/provider.go //go:generate mockgen -typed -destination=runtime/runtime.go -source=../pkg/runtime/runtime.go +//go:generate mockgen -typed -destination=runtime/provision/operations.go -source=../pkg/runtime/provision/operations.go package mock diff --git a/core/mock/runtime/provision/operations.go b/core/mock/runtime/provision/operations.go new file mode 100644 index 00000000..6f9d0ddd --- /dev/null +++ b/core/mock/runtime/provision/operations.go @@ -0,0 +1,197 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ../pkg/runtime/provision/operations.go +// +// Generated by this command: +// +// mockgen -typed -destination=runtime/provision/operations.go -source=../pkg/runtime/provision/operations.go +// + +// Package mock_provision is a generated GoMock package. +package mock_provision + +import ( + reflect "reflect" + + provision "drassi.run/core/pkg/runtime/provision" + sandboxer "drassi.run/core/pkg/sandboxer" + gomock "go.uber.org/mock/gomock" +) + +// MockOperation is a mock of Operation interface. +type MockOperation[Req any] struct { + ctrl *gomock.Controller + recorder *MockOperationMockRecorder[Req] + isgomock struct{} +} + +// MockOperationMockRecorder is the mock recorder for MockOperation. +type MockOperationMockRecorder[Req any] struct { + mock *MockOperation[Req] +} + +// NewMockOperation creates a new mock instance. +func NewMockOperation[Req any](ctrl *gomock.Controller) *MockOperation[Req] { + mock := &MockOperation[Req]{ctrl: ctrl} + mock.recorder = &MockOperationMockRecorder[Req]{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockOperation[Req]) EXPECT() *MockOperationMockRecorder[Req] { + return m.recorder +} + +// Name mocks base method. +func (m *MockOperation[Req]) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockOperationMockRecorder[Req]) Name() *MockOperationNameCall[Req] { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockOperation[Req])(nil).Name)) + return &MockOperationNameCall[Req]{Call: call} +} + +// MockOperationNameCall wrap *gomock.Call +type MockOperationNameCall[Req any] struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockOperationNameCall[Req]) Return(arg0 string) *MockOperationNameCall[Req] { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockOperationNameCall[Req]) Do(f func() string) *MockOperationNameCall[Req] { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockOperationNameCall[Req]) DoAndReturn(f func() string) *MockOperationNameCall[Req] { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// PostLaunch mocks base method. +func (m *MockOperation[Req]) PostLaunch(pctx *provision.Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PostLaunch", pctx, sb) + ret0, _ := ret[0].(sandboxer.Sandbox) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PostLaunch indicates an expected call of PostLaunch. +func (mr *MockOperationMockRecorder[Req]) PostLaunch(pctx, sb any) *MockOperationPostLaunchCall[Req] { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostLaunch", reflect.TypeOf((*MockOperation[Req])(nil).PostLaunch), pctx, sb) + return &MockOperationPostLaunchCall[Req]{Call: call} +} + +// MockOperationPostLaunchCall wrap *gomock.Call +type MockOperationPostLaunchCall[Req any] struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockOperationPostLaunchCall[Req]) Return(arg0 sandboxer.Sandbox, arg1 error) *MockOperationPostLaunchCall[Req] { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockOperationPostLaunchCall[Req]) Do(f func(*provision.Context, sandboxer.Sandbox) (sandboxer.Sandbox, error)) *MockOperationPostLaunchCall[Req] { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockOperationPostLaunchCall[Req]) DoAndReturn(f func(*provision.Context, sandboxer.Sandbox) (sandboxer.Sandbox, error)) *MockOperationPostLaunchCall[Req] { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// PreLaunch mocks base method. +func (m *MockOperation[Req]) PreLaunch(pctx *provision.Context, req Req) (Req, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PreLaunch", pctx, req) + ret0, _ := ret[0].(Req) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PreLaunch indicates an expected call of PreLaunch. +func (mr *MockOperationMockRecorder[Req]) PreLaunch(pctx, req any) *MockOperationPreLaunchCall[Req] { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreLaunch", reflect.TypeOf((*MockOperation[Req])(nil).PreLaunch), pctx, req) + return &MockOperationPreLaunchCall[Req]{Call: call} +} + +// MockOperationPreLaunchCall wrap *gomock.Call +type MockOperationPreLaunchCall[Req any] struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockOperationPreLaunchCall[Req]) Return(arg0 Req, arg1 error) *MockOperationPreLaunchCall[Req] { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockOperationPreLaunchCall[Req]) Do(f func(*provision.Context, Req) (Req, error)) *MockOperationPreLaunchCall[Req] { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockOperationPreLaunchCall[Req]) DoAndReturn(f func(*provision.Context, Req) (Req, error)) *MockOperationPreLaunchCall[Req] { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Prepare mocks base method. +func (m *MockOperation[Req]) Prepare(pctx *provision.Context) (sandboxer.Cleanup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Prepare", pctx) + ret0, _ := ret[0].(sandboxer.Cleanup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Prepare indicates an expected call of Prepare. +func (mr *MockOperationMockRecorder[Req]) Prepare(pctx any) *MockOperationPrepareCall[Req] { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prepare", reflect.TypeOf((*MockOperation[Req])(nil).Prepare), pctx) + return &MockOperationPrepareCall[Req]{Call: call} +} + +// MockOperationPrepareCall wrap *gomock.Call +type MockOperationPrepareCall[Req any] struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockOperationPrepareCall[Req]) Return(arg0 sandboxer.Cleanup, arg1 error) *MockOperationPrepareCall[Req] { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockOperationPrepareCall[Req]) Do(f func(*provision.Context) (sandboxer.Cleanup, error)) *MockOperationPrepareCall[Req] { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockOperationPrepareCall[Req]) DoAndReturn(f func(*provision.Context) (sandboxer.Cleanup, error)) *MockOperationPrepareCall[Req] { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/core/pkg/runtime/provision/context_test.go b/core/pkg/runtime/provision/context_test.go index 361a3315..da98f547 100644 --- a/core/pkg/runtime/provision/context_test.go +++ b/core/pkg/runtime/provision/context_test.go @@ -14,66 +14,74 @@ import ( "drassi.run/core/config" "drassi.run/core/pkg/runtime/provision" ocistore "drassi.run/core/pkg/store/oci" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) -func TestContextGenericState(t *testing.T) { - ctx := context.Background() +func TestContextSuite(t *testing.T) { + suite.Run(t, new(ContextTestSuite)) +} + +type ContextTestSuite struct { + suite.Suite +} + +func (s *ContextTestSuite) TestGenericState() { + ctx := s.T().Context() rtCfg := &config.Runtime{Image: "drassi/node:24"} pctx := provision.NewContext(ctx, "node", rtCfg, "/opt/drassi/runtimes/node") - require.Equal(t, "node", pctx.RuntimeName) - require.Equal(t, "/opt/drassi/runtimes/node", pctx.TargetDir) - require.Equal(t, rtCfg, pctx.Config) + s.Require().Equal("node", pctx.RuntimeName) + s.Require().Equal("/opt/drassi/runtimes/node", pctx.TargetDir) + s.Require().Equal(rtCfg, pctx.Config) // Test typed key string pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") val, ok := pctx.Get(provision.KeyHostMountDir) - require.True(t, ok) - require.Equal(t, "/var/lib/drassi/storage/overlay/merged", val) - require.Equal(t, "/var/lib/drassi/storage/overlay/merged", pctx.MustGet(provision.KeyHostMountDir)) + s.Require().True(ok) + s.Require().Equal("/var/lib/drassi/storage/overlay/merged", val) + s.Require().Equal("/var/lib/drassi/storage/overlay/merged", pctx.MustGet(provision.KeyHostMountDir)) // Test KeyMountID pctx.Set(provision.KeyMountID, "mount-12345") mountID, ok := pctx.Get(provision.KeyMountID) - require.True(t, ok) - require.Equal(t, "mount-12345", mountID) - require.Equal(t, "mount-12345", pctx.MustGet(provision.KeyMountID)) + s.Require().True(ok) + s.Require().Equal("mount-12345", mountID) + s.Require().Equal("mount-12345", pctx.MustGet(provision.KeyMountID)) // Test KeyImage img := &ocistore.Image{ID: "img-node"} pctx.Set(provision.KeyImage, img) gotImg, ok := pctx.Get(provision.KeyImage) - require.True(t, ok) - require.Equal(t, img, gotImg) - require.Equal(t, img, pctx.MustGet(provision.KeyImage)) + s.Require().True(ok) + s.Require().Equal(img, gotImg) + s.Require().Equal(img, pctx.MustGet(provision.KeyImage)) // Test missing key const keyMissing = provision.StateKey[int]("missing_key") intVal, ok := pctx.Get(keyMissing) - require.False(t, ok) - require.Equal(t, 0, intVal) - require.Panics(t, func() { + s.Require().False(ok) + s.Require().Equal(0, intVal) + s.Require().Panics(func() { pctx.MustGet(keyMissing) }) // Test type mismatch const keyMismatch = provision.StateKey[int]("host_mount_dir") mismatchVal, ok := pctx.Get(keyMismatch) - require.False(t, ok) - require.Equal(t, 0, mismatchVal) - require.Panics(t, func() { + s.Require().False(ok) + s.Require().Equal(0, mismatchVal) + s.Require().Panics(func() { pctx.MustGet(keyMismatch) }) } -func TestContextCancellation(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) +func (s *ContextTestSuite) TestCancellation() { + ctx, cancel := context.WithCancel(s.T().Context()) pctx := provision.NewContext(ctx, "node", nil, "") select { case <-pctx.Done(): - t.Fatal("context should not be done yet") + s.T().Fatal("context should not be done yet") default: } @@ -81,8 +89,8 @@ func TestContextCancellation(t *testing.T) { select { case <-pctx.Done(): - require.Equal(t, context.Canceled, pctx.Err()) + s.Require().Equal(context.Canceled, pctx.Err()) case <-time.After(time.Second): - t.Fatal("timed out waiting for context cancellation") + s.T().Fatal("timed out waiting for context cancellation") } } diff --git a/core/pkg/runtime/provision/operations.go b/core/pkg/runtime/provision/operations.go index edce3a7c..4bf256da 100644 --- a/core/pkg/runtime/provision/operations.go +++ b/core/pkg/runtime/provision/operations.go @@ -7,7 +7,9 @@ package provision import ( + "context" "fmt" + "slices" "drassi.run/core/pkg/sandboxer" "drassi.run/core/pkg/store/oci" @@ -19,90 +21,78 @@ const ( KeyImage = StateKey[*ocistore.Image]("image") ) -type Operation interface { +type Operation[Req any] interface { Name() string - PreLaunch(pctx *Context) error - PostLaunch(pctx *Context, sb sandboxer.Sandbox) error + Prepare(pctx *Context) (sandboxer.Cleanup, error) + PreLaunch(pctx *Context, req Req) (Req, error) + PostLaunch(pctx *Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) } -type Noop struct{} +type Noop[Req any] struct{} -func (op Noop) Name() string { return "noop" } -func (op Noop) PreLaunch(_ *Context) error { return nil } -func (op Noop) PostLaunch(_ *Context, _ sandboxer.Sandbox) error { return nil } - -type OpFunc struct { - PreFunc func(ctx *Context) error - PostFunc func(ctx *Context, sb sandboxer.Sandbox) error -} - -func (op *OpFunc) Name() string { return "func" } - -func (op *OpFunc) PreLaunch(pctx *Context) error { - if fn := op.PreFunc; fn != nil { - return fn(pctx) - } - return nil +func (Noop[Req]) Name() string { return "noop" } +func (Noop[Req]) Prepare(_ *Context) (sandboxer.Cleanup, error) { return nil, nil } +func (Noop[Req]) PreLaunch(_ *Context, req Req) (Req, error) { return req, nil } +func (Noop[Req]) PostLaunch(_ *Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) { + return sb, nil } -func (op *OpFunc) PostLaunch(pctx *Context, sb sandboxer.Sandbox) error { - if fn := op.PostFunc; fn != nil { - return fn(pctx, sb) - } - return nil -} - -type pullOp struct { - Noop +type pullOp[Req any] struct { + Noop[Req] store ocistore.Manager } // Pull returns an Operation that checks if the configured image is locally available, // and pulls it using the provided ocistore.Manager if missing. -func Pull(store ocistore.Manager) Operation { - return &pullOp{store: store} +func Pull[Req any](store ocistore.Manager) Operation[Req] { + return &pullOp[Req]{store: store} } -func (op *pullOp) Name() string { return "pull" } +func (op *pullOp[Req]) Name() string { return "pull" } -func (op *pullOp) PreLaunch(pctx *Context) error { +func (op *pullOp[Req]) Prepare(pctx *Context) (sandboxer.Cleanup, error) { img, err := op.store.Image(pctx, pctx.Config.Image) if err != nil { - return fmt.Errorf("check image %q: %w", pctx.Config.Image, err) + return nil, fmt.Errorf("check image %q: %w", pctx.Config.Image, err) } if img == nil { if img, err = op.store.Pull(pctx, pctx.Config.Image); err != nil { - return fmt.Errorf("pull image %q: %w", pctx.Config.Image, err) + return nil, fmt.Errorf("pull image %q: %w", pctx.Config.Image, err) } } pctx.Set(KeyImage, img) - return nil + return nil, nil } -type mountOp struct { - Noop +type mountOp[Req any] struct { + Noop[Req] store ocistore.Manager opts []ocistore.MountOption } // Mount returns an Operation that mounts the configured runtime image with the given MountOptions, -// and records KeyHostMountDir and KeyMountID in the Context. -func Mount(store ocistore.Manager, opts ...ocistore.MountOption) Operation { - return &mountOp{store: store, opts: opts} +// records KeyHostMountDir and KeyMountID in Context, and returns an unmount Cleanup closure. +func Mount[Req any](store ocistore.Manager, opts ...ocistore.MountOption) Operation[Req] { + return &mountOp[Req]{store: store, opts: opts} } -func (op *mountOp) Name() string { return "mount" } +func (op *mountOp[Req]) Name() string { return "mount" } -func (op *mountOp) PreLaunch(pctx *Context) error { +func (op *mountOp[Req]) Prepare(pctx *Context) (sandboxer.Cleanup, error) { img, ok := pctx.Get(KeyImage) if !ok || img == nil { - return fmt.Errorf("image %q not found in context: pull operation must be used first", pctx.Config.Image) + return nil, fmt.Errorf("image %q not found in context: pull operation must be used first", pctx.Config.Image) } - mountDir, id, err := op.store.Mount(pctx, img, op.opts...) + opts := append(slices.Clone(op.opts), ocistore.WithWritable(!pctx.Config.ReadOnly)) + mountDir, id, err := op.store.Mount(pctx, img, opts...) if err != nil { - return fmt.Errorf("mount image %q: %w", pctx.Config.Image, err) + return nil, fmt.Errorf("mount image %q: %w", pctx.Config.Image, err) } pctx.Set(KeyHostMountDir, mountDir) pctx.Set(KeyMountID, id) - return nil + + cleanup := func(ctx context.Context) error { + return op.store.Unmount(ctx, id) + } + return cleanup, nil } diff --git a/core/pkg/runtime/provision/operations_test.go b/core/pkg/runtime/provision/operations_test.go index 070f28e4..99ee69ee 100644 --- a/core/pkg/runtime/provision/operations_test.go +++ b/core/pkg/runtime/provision/operations_test.go @@ -8,189 +8,120 @@ package provision_test import ( "context" - "errors" "testing" "drassi.run/core/config" - mock_ocistore "drassi.run/core/mock/store/oci" + mock_store "drassi.run/core/mock/store/oci" "drassi.run/core/pkg/runtime/provision" - "drassi.run/core/pkg/sandboxer" ocistore "drassi.run/core/pkg/store/oci" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) -func newTestSetup(t *testing.T, name, image string) (*mock_ocistore.MockManager, *provision.Context) { - t.Helper() +func TestPullOperation(t *testing.T) { ctrl := gomock.NewController(t) - mgr := mock_ocistore.NewMockManager(ctrl) - cfg := &config.Runtime{ - Image: image, - Paths: []string{"bin"}, - } - pctx := provision.NewContext(context.Background(), name, cfg, "/opt/drassi/runtimes/"+name) - return mgr, pctx -} - -func TestSharedOperations(t *testing.T) { - mgr, pctx := newTestSetup(t, "node", "drassi/node:24") - img := &ocistore.Image{ID: "img-node", Names: []string{"drassi/node:24"}} + store := mock_store.NewMockManager(ctrl) - mgr.EXPECT().Image(pctx, "drassi/node:24").Return(nil, nil) - mgr.EXPECT().Pull(pctx, "drassi/node:24").Return(img, nil) - mgr.EXPECT().Mount(pctx, img, gomock.Any()).Return("/var/lib/drassi/storage/overlay/merged", "layer-123", nil) + op := provision.Pull[any](store) + require.Equal(t, "pull", op.Name()) - p := provision.NewPipeline( - provision.Pull(mgr), - provision.Mount(mgr, ocistore.WithWritable(true)), - ) + rtCfg := &config.Runtime{Image: "drassi/node:24"} + pctx := provision.NewContext(t.Context(), "node", rtCfg, "/opt/drassi/runtimes/node") - err := p.PreLaunch(pctx) - require.NoError(t, err) - require.Equal(t, "/var/lib/drassi/storage/overlay/merged", pctx.MustGet(provision.KeyHostMountDir)) - require.Equal(t, "layer-123", pctx.MustGet(provision.KeyMountID)) - require.Equal(t, img, pctx.MustGet(provision.KeyImage)) + t.Run("image cached locally", func(t *testing.T) { + expectedImg := &ocistore.Image{} + store.EXPECT().Image(pctx, "drassi/node:24").Return(expectedImg, nil) - err = p.PostLaunch(pctx, nil) - require.NoError(t, err) -} + cleanup, err := op.Prepare(pctx) + require.NoError(t, err) + require.Nil(t, cleanup) -func TestPullOperation(t *testing.T) { - t.Run("operation metadata", func(t *testing.T) { - mgr, _ := newTestSetup(t, "python", "python:3.12") - op := provision.Pull(mgr) - require.Equal(t, "pull", op.Name()) - require.NoError(t, op.PostLaunch(nil, nil)) + img, ok := pctx.Get(provision.KeyImage) + require.True(t, ok) + require.Equal(t, expectedImg, img) }) - t.Run("image already exists - pull skipped", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "python", "python:3.12") - img := &ocistore.Image{ID: "img-existing", Names: []string{"python:3.12"}} - - mgr.EXPECT().Image(pctx, "python:3.12").Return(img, nil) + t.Run("image not cached - pulls successfully", func(t *testing.T) { + expectedImg := &ocistore.Image{} + store.EXPECT().Image(pctx, "drassi/node:24").Return(nil, nil) + store.EXPECT().Pull(pctx, "drassi/node:24").Return(expectedImg, nil) - op := provision.Pull(mgr) - err := op.PreLaunch(pctx) + cleanup, err := op.Prepare(pctx) require.NoError(t, err) - require.Equal(t, img, pctx.MustGet(provision.KeyImage)) - }) - - t.Run("image does not exist - pull triggered", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "python", "python:3.12") - img := &ocistore.Image{ID: "img-new", Names: []string{"python:3.12"}} + require.Nil(t, cleanup) - mgr.EXPECT().Image(pctx, "python:3.12").Return(nil, nil) - mgr.EXPECT().Pull(pctx, "python:3.12").Return(img, nil) - - op := provision.Pull(mgr) - err := op.PreLaunch(pctx) - require.NoError(t, err) - require.Equal(t, img, pctx.MustGet(provision.KeyImage)) + img, ok := pctx.Get(provision.KeyImage) + require.True(t, ok) + require.Equal(t, expectedImg, img) }) +} - t.Run("image check error", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "python", "python:3.12") - - mgr.EXPECT().Image(pctx, "python:3.12").Return(nil, errors.New("storage backend disconnected")) - - op := provision.Pull(mgr) - err := op.PreLaunch(pctx) - require.Error(t, err) - require.Contains(t, err.Error(), `check image "python:3.12": storage backend disconnected`) - }) +func TestMountOperation(t *testing.T) { + ctrl := gomock.NewController(t) + store := mock_store.NewMockManager(ctrl) - t.Run("pull error", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "python", "python:3.12") + op := provision.Mount[any](store, ocistore.WithWritable(true)) + require.Equal(t, "mount", op.Name()) - mgr.EXPECT().Image(pctx, "python:3.12").Return(nil, nil) - mgr.EXPECT().Pull(pctx, "python:3.12").Return(nil, errors.New("network timeout")) + rtCfg := &config.Runtime{Image: "drassi/node:24"} + pctx := provision.NewContext(t.Context(), "node", rtCfg, "/opt/drassi/runtimes/node") - op := provision.Pull(mgr) - err := op.PreLaunch(pctx) + t.Run("missing image returns error", func(t *testing.T) { + cleanup, err := op.Prepare(pctx) require.Error(t, err) - require.Contains(t, err.Error(), `pull image "python:3.12": network timeout`) + require.Nil(t, cleanup) + require.Contains(t, err.Error(), "image \"drassi/node:24\" not found in context") }) -} -func TestMountOperation(t *testing.T) { - t.Run("operation metadata", func(t *testing.T) { - mgr, _ := newTestSetup(t, "node", "node:24") - op := provision.Mount(mgr) - require.Equal(t, "mount", op.Name()) - require.NoError(t, op.PostLaunch(nil, nil)) - }) - - t.Run("mount success with options", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "node", "node:24") - img := &ocistore.Image{ID: "img-node", Names: []string{"node:24"}} + t.Run("mounts image and returns unmount cleanup", func(t *testing.T) { + img := &ocistore.Image{} pctx.Set(provision.KeyImage, img) - mgr.EXPECT().Mount(pctx, img, gomock.Any()).DoAndReturn(func(_ context.Context, _ *ocistore.Image, opts ...ocistore.MountOption) (string, string, error) { - require.Len(t, opts, 1) - return "/var/lib/drassi/storage/overlay/merged", "layer-123", nil - }) + store.EXPECT().Mount(pctx, img, gomock.Any()).Return("/var/lib/drassi/mount", "layer-123", nil) - op := provision.Mount(mgr, ocistore.WithWritable(true)) - err := op.PreLaunch(pctx) + cleanup, err := op.Prepare(pctx) require.NoError(t, err) - require.Equal(t, "/var/lib/drassi/storage/overlay/merged", pctx.MustGet(provision.KeyHostMountDir)) - require.Equal(t, "layer-123", pctx.MustGet(provision.KeyMountID)) - require.Equal(t, img, pctx.MustGet(provision.KeyImage)) - }) - - t.Run("image not found in context error", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "node", "node:24") - op := provision.Mount(mgr) - - err := op.PreLaunch(pctx) - require.Error(t, err) - require.Contains(t, err.Error(), `image "node:24" not found in context: pull operation must be used first`) - }) + require.NotNil(t, cleanup) - t.Run("mount error", func(t *testing.T) { - mgr, pctx := newTestSetup(t, "node", "node:24") - img := &ocistore.Image{ID: "img-node", Names: []string{"node:24"}} - pctx.Set(provision.KeyImage, img) - - mgr.EXPECT().Mount(pctx, img, gomock.Any()).Return("", "", errors.New("overlay mount failed")) + hostDir, ok := pctx.Get(provision.KeyHostMountDir) + require.True(t, ok) + require.Equal(t, "/var/lib/drassi/mount", hostDir) - op := provision.Mount(mgr) - err := op.PreLaunch(pctx) - require.Error(t, err) - require.Contains(t, err.Error(), `mount image "node:24": overlay mount failed`) + mountID, ok := pctx.Get(provision.KeyMountID) + require.True(t, ok) + require.Equal(t, "layer-123", mountID) - _, ok := pctx.Get(provision.KeyHostMountDir) - require.False(t, ok) - _, ok = pctx.Get(provision.KeyMountID) - require.False(t, ok) + // Executing cleanup unmounts layer + store.EXPECT().Unmount(gomock.Any(), "layer-123").Return(nil) + require.NoError(t, cleanup(t.Context())) }) -} -func TestOpFunc(t *testing.T) { - t.Run("metadata and default execution", func(t *testing.T) { - op := &provision.OpFunc{} - require.Equal(t, "func", op.Name()) - require.NoError(t, op.PreLaunch(nil)) - require.NoError(t, op.PostLaunch(nil, nil)) - }) + t.Run("readonly runtime mounts as non-writable", func(t *testing.T) { + roCfg := &config.Runtime{Image: "drassi/node:24", ReadOnly: true} + roCtx := provision.NewContext(t.Context(), "node", roCfg, "/opt/drassi/runtimes/node") + img := &ocistore.Image{} + roCtx.Set(provision.KeyImage, img) - t.Run("custom callbacks", func(t *testing.T) { - preCalled, postCalled := false, false - op := &provision.OpFunc{ - PreFunc: func(_ *provision.Context) error { - preCalled = true - return nil + store.EXPECT().Mount(roCtx, img, gomock.Any()).DoAndReturn( + func(_ context.Context, _ *ocistore.Image, opts ...ocistore.MountOption) (string, string, error) { + require.NotEmpty(t, opts) + return "/var/lib/drassi/mount-ro", "layer-ro", nil }, - PostFunc: func(_ *provision.Context, _ sandboxer.Sandbox) error { - postCalled = true - return nil - }, - } + ) + + cleanup, err := op.Prepare(roCtx) + require.NoError(t, err) + require.NotNil(t, cleanup) + + hostDir, ok := roCtx.Get(provision.KeyHostMountDir) + require.True(t, ok) + require.Equal(t, "/var/lib/drassi/mount-ro", hostDir) - require.NoError(t, op.PreLaunch(nil)) - require.True(t, preCalled) + mountID, ok := roCtx.Get(provision.KeyMountID) + require.True(t, ok) + require.Equal(t, "layer-ro", mountID) - require.NoError(t, op.PostLaunch(nil, nil)) - require.True(t, postCalled) + store.EXPECT().Unmount(gomock.Any(), "layer-ro").Return(nil) + require.NoError(t, cleanup(t.Context())) }) } diff --git a/core/pkg/runtime/provision/pipeline.go b/core/pkg/runtime/provision/pipeline.go deleted file mode 100644 index 81e5566d..00000000 --- a/core/pkg/runtime/provision/pipeline.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package provision - -import ( - "fmt" - "slices" - - "drassi.run/core/pkg/sandboxer" -) - -type Pipeline []Operation - -func NewPipeline(ops ...Operation) Pipeline { - return ops -} - -func (p Pipeline) PreLaunch(pctx *Context) error { - for _, op := range p { - if err := op.PreLaunch(pctx); err != nil { - return fmt.Errorf("operation %q pre-launch failed for runtime %q: %w", op.Name(), pctx.RuntimeName, err) - } - } - return nil -} - -func (p Pipeline) PostLaunch(pctx *Context, sb sandboxer.Sandbox) error { - for _, op := range slices.Backward(p) { - if err := op.PostLaunch(pctx, sb); err != nil { - return fmt.Errorf("operation %q post-launch failed for runtime %q: %w", op.Name(), pctx.RuntimeName, err) - } - } - return nil -} diff --git a/core/pkg/runtime/provision/pipeline_test.go b/core/pkg/runtime/provision/pipeline_test.go deleted file mode 100644 index 88217821..00000000 --- a/core/pkg/runtime/provision/pipeline_test.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package provision_test - -import ( - "context" - "errors" - "testing" - - "drassi.run/core/config" - "drassi.run/core/pkg/runtime/provision" - "drassi.run/core/pkg/sandboxer" - "github.com/stretchr/testify/require" -) - -func TestPipelineExecution(t *testing.T) { - var ( - op1PreCalled, op2PreCalled bool - op1PostCalled, op2PostCalled bool - op1PreCtx, op2PreCtx *provision.Context - op1PostCtx, op2PostCtx *provision.Context - ) - - op1 := &provision.OpFunc{ - PreFunc: func(pctx *provision.Context) error { - op1PreCalled = true - op1PreCtx = pctx - return nil - }, - PostFunc: func(pctx *provision.Context, _ sandboxer.Sandbox) error { - op1PostCalled = true - op1PostCtx = pctx - return nil - }, - } - op2 := &provision.OpFunc{ - PreFunc: func(pctx *provision.Context) error { - op2PreCalled = true - op2PreCtx = pctx - return nil - }, - PostFunc: func(pctx *provision.Context, _ sandboxer.Sandbox) error { - op2PostCalled = true - op2PostCtx = pctx - return nil - }, - } - - p := provision.NewPipeline(op1, op2) - pctx := provision.NewContext(context.Background(), "node", &config.Runtime{}, "/target") - - err := p.PreLaunch(pctx) - require.NoError(t, err) - require.True(t, op1PreCalled) - require.True(t, op2PreCalled) - require.Equal(t, pctx, op1PreCtx) - require.Equal(t, pctx, op2PreCtx) - - err = p.PostLaunch(pctx, nil) - require.NoError(t, err) - require.True(t, op1PostCalled) - require.True(t, op2PostCalled) - require.Equal(t, pctx, op1PostCtx) - require.Equal(t, pctx, op2PostCtx) -} - -func TestPipelinePreLaunchError(t *testing.T) { - op2PreCalled := false - op1 := &provision.OpFunc{ - PreFunc: func(_ *provision.Context) error { - return errors.New("pre fail") - }, - } - op2 := &provision.OpFunc{ - PreFunc: func(_ *provision.Context) error { - op2PreCalled = true - return nil - }, - } - - p := provision.NewPipeline(op1, op2) - pctx := provision.NewContext(context.Background(), "node", &config.Runtime{}, "/target") - - err := p.PreLaunch(pctx) - require.Error(t, err) - require.Contains(t, err.Error(), `operation "func" pre-launch failed for runtime "node": pre fail`) - require.False(t, op2PreCalled) -} - -func TestPipelinePostLaunchError(t *testing.T) { - op1PostCalled := false - op1 := &provision.OpFunc{ - PostFunc: func(_ *provision.Context, _ sandboxer.Sandbox) error { - op1PostCalled = true - return nil - }, - } - op2 := &provision.OpFunc{ - PostFunc: func(_ *provision.Context, _ sandboxer.Sandbox) error { - return errors.New("post fail") - }, - } - - p := provision.NewPipeline(op1, op2) - pctx := provision.NewContext(context.Background(), "python", &config.Runtime{}, "/target") - - err := p.PostLaunch(pctx, nil) - require.Error(t, err) - require.Contains(t, err.Error(), `operation "func" post-launch failed for runtime "python": post fail`) - require.False(t, op1PostCalled) -} - -func TestPipelineEmpty(t *testing.T) { - p := provision.NewPipeline() - pctx := provision.NewContext(context.Background(), "node", &config.Runtime{}, "/target") - - require.NoError(t, p.PreLaunch(pctx)) - require.NoError(t, p.PostLaunch(pctx, nil)) -} diff --git a/core/pkg/runtime/provision/provisioner.go b/core/pkg/runtime/provision/provisioner.go new file mode 100644 index 00000000..8b159dfd --- /dev/null +++ b/core/pkg/runtime/provision/provisioner.go @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package provision + +import ( + "cmp" + "context" + "fmt" + "path/filepath" + "slices" + "sync" + + "drassi.run/core/config" + "drassi.run/core/pkg/sandboxer" + "golang.org/x/sync/errgroup" +) + +// Launcher is a function that creates a sandbox from a provider-specific request. +type Launcher[Req any] func(ctx context.Context, req Req) (sandboxer.Sandbox, error) + +// Provisioner orchestrates multi-runtime provisioning as a Launcher decorator. +// It is completely stateless and safe for concurrent use across multiple launches. +type Provisioner[Req any] struct { + runtimes map[string]*config.Runtime + ops []Operation[Req] +} + +// New creates a new Provisioner with explicitly provided operations. Zero default ops are added. +func New[Req any]( + runtimes map[string]*config.Runtime, + ops ...Operation[Req], +) *Provisioner[Req] { + return &Provisioner[Req]{ + runtimes: runtimes, + ops: ops, + } +} + +// Launch decorates an inner Launcher with parallel preparation, sequential request transformation, +// inner execution, sequential post-launch actions, and automatic cleanup registration/rollback. +func (p *Provisioner[Req]) Launch(runtimeDir string, l Launcher[Req]) Launcher[Req] { + if p.Empty() { + return l + } + + return func(ctx context.Context, req Req) (sb sandboxer.Sandbox, err error) { + // 1. Deterministic ordering of runtimes by name + names := make([]string, 0, len(p.runtimes)) + for name := range p.runtimes { + names = append(names, name) + } + slices.Sort(names) + + contexts := make([]*Context, len(names)) + for i, name := range names { + targetDir := filepath.Join(runtimeDir, name) + contexts[i] = NewContext(ctx, name, p.runtimes[name], targetDir) + } + + var ( + mu sync.Mutex + cleanups []sandboxer.Cleanup + ) + + rollback := func(ctx context.Context) { + ctx = context.WithoutCancel(ctx) + for _, cleanup := range slices.Backward(cleanups) { + _ = cleanup(ctx) + } + } + + // 2. Parallel Prepare across all runtimes: parallel(pull -> mount) + g, groupCtx := errgroup.WithContext(ctx) + for _, pctx := range contexts { + pctx.Context = groupCtx + g.Go(func() error { + for _, op := range p.ops { + if c, err := op.Prepare(pctx); err != nil { + return fmt.Errorf("operation %q prepare failed for runtime %q: %w", op.Name(), pctx.RuntimeName, err) + } else if c != nil { + mu.Lock() + cleanups = append(cleanups, c) + mu.Unlock() + } + } + return nil + }) + } + if err = g.Wait(); err != nil { + rollback(ctx) + return + } + for _, pctx := range contexts { + pctx.Context = ctx + } + + // 3. Sequential PreLaunch request mutation + for _, pctx := range contexts { + for _, op := range p.ops { + if req, err = op.PreLaunch(pctx, req); err != nil { + rollback(ctx) + return nil, fmt.Errorf("operation %q pre-launch failed for runtime %q: %w", op.Name(), pctx.RuntimeName, err) + } + } + } + + // 4. Invoke inner launcher with mutated request + if sb, err = l(ctx, req); err != nil { + rollback(ctx) + return + } + + // 5. Sequential PostLaunch actions + for _, pctx := range contexts { + for _, op := range p.ops { + next, err := op.PostLaunch(pctx, sb) + if err != nil { + sb = cmp.Or(next, sb) + _ = sb.Terminate(context.WithoutCancel(ctx)) + rollback(ctx) + return nil, fmt.Errorf("operation %q post-launch failed for runtime %q: %w", op.Name(), pctx.RuntimeName, err) + } + sb = next + } + } + + // 6. Success: attach all cleanups to sandbox in LIFO order + if len(cleanups) > 0 { + slices.Reverse(cleanups) + sb = sandboxer.AddAfterCleanup(sb, cleanups...) + } + + return + } +} + +func (p *Provisioner[Req]) Empty() bool { + return p == nil || len(p.runtimes) == 0 || len(p.ops) == 0 +} diff --git a/core/pkg/runtime/provision/provisioner_test.go b/core/pkg/runtime/provision/provisioner_test.go new file mode 100644 index 00000000..f64852df --- /dev/null +++ b/core/pkg/runtime/provision/provisioner_test.go @@ -0,0 +1,327 @@ +/* + * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package provision_test + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "drassi.run/core/config" + mock_provision "drassi.run/core/mock/runtime/provision" + mock_sandboxer "drassi.run/core/mock/sandboxer" + "drassi.run/core/pkg/runtime/provision" + "drassi.run/core/pkg/sandboxer" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +func TestProvisionerSuite(t *testing.T) { + suite.Run(t, new(ProvisionerTestSuite)) +} + +type testRequest struct { + Items []string +} + +type ProvisionerTestSuite struct { + suite.Suite + ctrl *gomock.Controller + baseSb *mock_sandboxer.MockSandbox + op *mock_provision.MockOperation[*testRequest] +} + +func (s *ProvisionerTestSuite) SetupTest() { + s.ctrl = gomock.NewController(s.T()) + s.baseSb = mock_sandboxer.NewMockSandbox(s.ctrl) + s.op = mock_provision.NewMockOperation[*testRequest](s.ctrl) +} + +func (s *ProvisionerTestSuite) TestSuccess() { + var cleanedUp atomic.Bool + cleanupFn := func(context.Context) error { + cleanedUp.Store(true) + return nil + } + + s.op.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + return cleanupFn, nil + }).Times(2) + s.op.EXPECT().PreLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, req *testRequest) (*testRequest, error) { + s.Require().Equal("/opt/runtimes/"+pctx.RuntimeName, pctx.TargetDir) + req.Items = append(req.Items, pctx.RuntimeName) + return req, nil + }).Times(2) + s.op.EXPECT().PostLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) { + return sb, nil + }).Times(2) + + runtimes := map[string]*config.Runtime{ + "node": {}, + "python": {}, + } + + p := provision.New[*testRequest]( + runtimes, + s.op, + ) + + innerLauncherCalled := false + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + innerLauncherCalled = true + s.Require().Equal([]string{"node", "python"}, req.Items) // sorted order + return s.baseSb, nil + } + + req := &testRequest{} + decorated := p.Launch("/opt/runtimes", innerLauncher) + sb, err := decorated(s.T().Context(), req) + s.Require().NoError(err) + s.Require().True(innerLauncherCalled) + s.Require().NotNil(sb) + + // Verify cleanup was attached to sandbox + s.baseSb.EXPECT().Terminate(gomock.Any()).Return(nil) + s.Require().NoError(sb.Terminate(s.T().Context())) + s.Require().True(cleanedUp.Load()) +} + +func (s *ProvisionerTestSuite) TestPrepareFailureRollback() { + var cleanup1Called atomic.Bool + + s.op.EXPECT().Name().Return("fail-op").AnyTimes() + s.op.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + if pctx.RuntimeName == "node" { + return func(context.Context) error { + cleanup1Called.Store(true) + return nil + }, nil + } + return nil, errors.New("pull failed") + }).Times(2) + + runtimes := map[string]*config.Runtime{ + "node": {}, + "python": {}, + } + + p := provision.New[*testRequest](runtimes, s.op) + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + s.T().Fatal("inner launcher should not be called on prepare failure") + return nil, nil + } + + _, err := p.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().Error(err) + s.Require().Contains(err.Error(), "pull failed") + s.Require().True(cleanup1Called.Load(), "rollback should execute cleanup for succeeded runtimes") +} + +func (s *ProvisionerTestSuite) TestInnerLauncherFailureRollback() { + var cleanupCalled atomic.Bool + + s.op.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + return func(context.Context) error { + cleanupCalled.Store(true) + return nil + }, nil + }).Times(1) + s.op.EXPECT().PreLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, req *testRequest) (*testRequest, error) { + return req, nil + }).Times(1) + + runtimes := map[string]*config.Runtime{"node": {}} + p := provision.New[*testRequest](runtimes, s.op) + + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + return nil, errors.New("container run error") + } + + _, err := p.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().Error(err) + s.Require().Contains(err.Error(), "container run error") + s.Require().True(cleanupCalled.Load(), "rollback should execute cleanups when inner launcher fails") +} + +func (s *ProvisionerTestSuite) TestPreLaunchFailureRollback() { + var cleanupCalled atomic.Bool + + s.op.EXPECT().Name().Return("fail-op").AnyTimes() + s.op.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + return func(context.Context) error { + cleanupCalled.Store(true) + return nil + }, nil + }).Times(1) + s.op.EXPECT().PreLaunch(gomock.Any(), gomock.Any()).Return(nil, errors.New("prelaunch mutation failed")).Times(1) + + runtimes := map[string]*config.Runtime{"node": {}} + p := provision.New[*testRequest](runtimes, s.op) + + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + s.T().Fatal("inner launcher should not be called on prelaunch failure") + return nil, nil + } + + _, err := p.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().Error(err) + s.Require().Contains(err.Error(), "prelaunch mutation failed") + s.Require().True(cleanupCalled.Load(), "rollback should execute cleanups when pre-launch fails") +} + +func (s *ProvisionerTestSuite) TestPostLaunchFailureRollback() { + var cleanupCalled atomic.Bool + + s.op.EXPECT().Name().Return("fail-op").AnyTimes() + s.op.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + return func(context.Context) error { + cleanupCalled.Store(true) + return nil + }, nil + }).Times(1) + s.op.EXPECT().PreLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, req *testRequest) (*testRequest, error) { + return req, nil + }).Times(1) + s.op.EXPECT().PostLaunch(gomock.Any(), gomock.Any()).Return(nil, errors.New("postlaunch failed")).Times(1) + + runtimes := map[string]*config.Runtime{"node": {}} + p := provision.New[*testRequest](runtimes, s.op) + + s.baseSb.EXPECT().Terminate(gomock.Any()).Return(nil) + + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + return s.baseSb, nil + } + + _, err := p.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().Error(err) + s.Require().Contains(err.Error(), "postlaunch failed") + s.Require().True(cleanupCalled.Load(), "rollback should execute cleanups when post-launch fails") +} + +func (s *ProvisionerTestSuite) TestPassthrough() { + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + return s.baseSb, nil + } + + // 1. Nil provisioner + var pNil *provision.Provisioner[*testRequest] + sb, err := pNil.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().NoError(err) + s.Require().Equal(s.baseSb, sb) + + // 2. Empty runtimes + pEmptyRuntimes := provision.New[*testRequest](nil) + sb, err = pEmptyRuntimes.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().NoError(err) + s.Require().Equal(s.baseSb, sb) + + // 3. Empty ops + pEmptyOps := provision.New[*testRequest](map[string]*config.Runtime{"node": {}}) + sb, err = pEmptyOps.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().NoError(err) + s.Require().Equal(s.baseSb, sb) +} + +func (s *ProvisionerTestSuite) TestLIFOCleanupOrder() { + var mu sync.Mutex + var order []int + + op1 := mock_provision.NewMockOperation[*testRequest](s.ctrl) + op1.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + return func(ctx context.Context) error { + mu.Lock() + order = append(order, 1) + mu.Unlock() + return nil + }, nil + }).Times(1) + op1.EXPECT().PreLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, req *testRequest) (*testRequest, error) { + return req, nil + }).Times(1) + op1.EXPECT().PostLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) { + return sb, nil + }).Times(1) + + op2 := mock_provision.NewMockOperation[*testRequest](s.ctrl) + op2.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + return func(ctx context.Context) error { + mu.Lock() + order = append(order, 2) + mu.Unlock() + return nil + }, nil + }).Times(1) + op2.EXPECT().PreLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, req *testRequest) (*testRequest, error) { + return req, nil + }).Times(1) + op2.EXPECT().PostLaunch(gomock.Any(), gomock.Any()).DoAndReturn(func(pctx *provision.Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) { + return sb, nil + }).Times(1) + + p := provision.New[*testRequest]( + map[string]*config.Runtime{"node": {}}, + op1, + op2, + ) + + s.baseSb.EXPECT().Terminate(gomock.Any()).Return(nil) + innerLauncher := func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + return s.baseSb, nil + } + + sb, err := p.Launch("/opt/runtimes", innerLauncher)(s.T().Context(), &testRequest{}) + s.Require().NoError(err) + + s.Require().NoError(sb.Terminate(s.T().Context())) + + mu.Lock() + defer mu.Unlock() + s.Require().Equal([]int{2, 1}, order, "cleanups should execute in LIFO order (reverse of registration)") +} + +func (s *ProvisionerTestSuite) TestPrepareCancellation() { + canceled := make(chan struct{}) + + s.op.EXPECT().Name().Return("cancel-test-op").AnyTimes() + s.op.EXPECT().Prepare(gomock.Any()).DoAndReturn(func(pctx *provision.Context) (sandboxer.Cleanup, error) { + if pctx.RuntimeName == "fast-fail" { + return nil, errors.New("boom") + } + // sibling runtime should see cancellation + select { + case <-pctx.Done(): + close(canceled) + case <-time.After(2 * time.Second): + s.T().Error("timed out waiting for sibling context cancellation") + } + return nil, nil + }).Times(2) + + p := provision.New[*testRequest]( + map[string]*config.Runtime{ + "fast-fail": {}, + "slow": {}, + }, + s.op, + ) + + _, err := p.Launch("/opt/runtimes", func(ctx context.Context, req *testRequest) (sandboxer.Sandbox, error) { + return nil, nil + })(s.T().Context(), &testRequest{}) + + s.Require().Error(err) + select { + case <-canceled: + // success: sibling was canceled + default: + s.T().Fatal("expected sibling runtime to receive context cancellation") + } +} diff --git a/core/pkg/sandboxer/container/engine.go b/core/pkg/sandboxer/container/engine.go index 0f3eb77b..fbfdb1d2 100644 --- a/core/pkg/sandboxer/container/engine.go +++ b/core/pkg/sandboxer/container/engine.go @@ -8,6 +8,7 @@ package container import ( "context" + "errors" "fmt" "maps" "strconv" @@ -21,6 +22,7 @@ import ( "drassi.run/core/pkg/container/types" "drassi.run/core/pkg/model/records" "drassi.run/core/pkg/model/workflows" + "drassi.run/core/pkg/runtime/provision" "drassi.run/core/pkg/sandboxer" "drassi.run/core/pkg/store/oci" "drassi.run/core/pkg/stream" @@ -54,8 +56,11 @@ type factory struct { runtimes map[string]*config.Runtime } -func (f *factory) ProvisionRuntime(store ocistore.Manager, config map[string]*config.Runtime) { +func (f *factory) SetOciStore(store ocistore.Manager) { f.store = store +} + +func (f *factory) ProvisionRuntime(config map[string]*config.Runtime) { f.runtimes = config } @@ -64,7 +69,19 @@ func (f *factory) Create() (sandboxer.Engine, error) { } func (f *factory) doCreate() (sandboxer.Engine, error) { - return New(f.cfg) + var prov *provision.Provisioner[*types.ContainerSpec] + if len(f.runtimes) > 0 { + if f.store == nil { + return nil, errors.New("oci store is required when runtimes are configured") + } + prov = provision.New[*types.ContainerSpec]( + f.runtimes, + provision.Pull[*types.ContainerSpec](f.store), + provision.Mount[*types.ContainerSpec](f.store), + AddBindMount(), + ) + } + return New(f.cfg, prov) } type Bootstrapper interface { @@ -80,9 +97,10 @@ type Config struct { type engine struct { client container.Engine defaultImage string + provisioner *provision.Provisioner[*types.ContainerSpec] } -func New(config *Config) (sandboxer.Engine, error) { +func New(config *Config, prov *provision.Provisioner[*types.ContainerSpec]) (sandboxer.Engine, error) { if config.Implementation != "docker" { return nil, fmt.Errorf("unsupported container implementation: %s", config.Implementation) } @@ -99,21 +117,21 @@ func New(config *Config) (sandboxer.Engine, error) { } client = container.WithTelemetry(client) - e := &engine{ + return NewWithClient(client, config.Image, prov), nil +} + +func NewWithClient(client container.Engine, defaultImage string, prov *provision.Provisioner[*types.ContainerSpec]) sandboxer.Engine { + return &engine{ client: client, - defaultImage: config.Image, + defaultImage: defaultImage, + provisioner: prov, } - return e, nil } func NewBootstrapper(client container.Engine) Bootstrapper { return &engine{client: client} } -func (e *engine) Close() error { - return e.client.Close() -} - func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*sandboxer.LaunchResponse, error) { var ( sb sandboxer.Sandbox @@ -122,21 +140,19 @@ func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*san if req.JobContainer == nil { spec := &types.ContainerSpec{ - Image: e.defaultImage, - Entrypoint: []string{"sleep"}, - Command: []string{"infinity"}, + Image: e.defaultImage, + Entrypoint: []string{"sleep"}, + Command: []string{"infinity"}, + NetworkMode: "host", } - spec.NetworkMode = "host" - runOpts := &container.RunOptions{ - Stdio: new(types.Stdio), - Streams: new(stream.Streams), + + var err error + launcher := e.launch + if prov := e.provisioner; prov != nil { + launcher = prov.Launch(defaultLayout.Runtimes, launcher) } - if cid, err := e.client.ContainerRun(ctx, spec, runOpts); err != nil { + if sb, err = launcher(ctx, spec); err != nil { return nil, err - } else if sb, err = newSandbox(ctx, e.client, cid); err != nil { - return nil, err - } else { - containerId = cid } } @@ -144,6 +160,11 @@ func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*san if err != nil { return nil, err } + if req.JobContainer == nil && sb != nil { + if s, ok := sandboxer.Unwrap(sb).(*sandbox); ok { + containerId = s.containerId + } + } if containerId != "" { resp.JobContainer = &records.ContainerInfo{ Id: containerId, @@ -152,6 +173,18 @@ func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*san return resp, nil } +func (e *engine) launch(ctx context.Context, spec *types.ContainerSpec) (sandboxer.Sandbox, error) { + runOpts := &container.RunOptions{ + Stdio: new(types.Stdio), + Streams: new(stream.Streams), + } + cid, err := e.client.ContainerRun(ctx, spec, runOpts) + if err != nil { + return nil, err + } + return newSandbox(ctx, e.client, cid) +} + func (e *engine) Bootstrap(ctx context.Context, sb sandboxer.Sandbox, req *sandboxer.LaunchRequest) (resp *sandboxer.LaunchResponse, err error) { resp = &sandboxer.LaunchResponse{ Sandbox: sb, @@ -247,6 +280,10 @@ func (e *engine) Bootstrap(ctx context.Context, sb sandboxer.Sandbox, req *sandb return resp, nil } +func (e *engine) Close() error { + return e.client.Close() +} + func (e *engine) parseContainer(def *workflows.Container, refiners []refiner) (spec *types.ContainerSpec, err error) { if spec, _, err = cli.Parse(def.Options); err != nil { return diff --git a/core/pkg/sandboxer/container/engine_test.go b/core/pkg/sandboxer/container/engine_test.go new file mode 100644 index 00000000..e6c23744 --- /dev/null +++ b/core/pkg/sandboxer/container/engine_test.go @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package container + +import ( + "context" + "testing" + + "drassi.run/core/config" + mock_container "drassi.run/core/mock/container" + mock_store "drassi.run/core/mock/store/oci" + "drassi.run/core/pkg/container" + "drassi.run/core/pkg/container/types" + "drassi.run/core/pkg/runtime/provision" + "drassi.run/core/pkg/sandboxer" + ocistore "drassi.run/core/pkg/store/oci" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +func TestContainerEngineSuite(t *testing.T) { + suite.Run(t, new(ContainerEngineTestSuite)) +} + +type ContainerEngineTestSuite struct { + suite.Suite + ctrl *gomock.Controller + store *mock_store.MockManager + mockClient *mock_container.MockEngine +} + +func (s *ContainerEngineTestSuite) SetupTest() { + s.ctrl = gomock.NewController(s.T()) + s.store = mock_store.NewMockManager(s.ctrl) + s.mockClient = mock_container.NewMockEngine(s.ctrl) +} + +func (s *ContainerEngineTestSuite) mockContainerLifecycle(containerID string, validateSpec func(spec *types.ContainerSpec)) { + s.mockClient.EXPECT().ContainerRun(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, spec *types.ContainerSpec, _ *container.RunOptions) (string, error) { + if validateSpec != nil { + validateSpec(spec) + } + return containerID, nil + }, + ) + s.mockClient.EXPECT().CopyIn(gomock.Any(), containerID, gomock.Any()).Return(nil) + s.mockClient.EXPECT().ContainerInspect(gomock.Any(), containerID).Return(&types.ContainerSpec{}, nil) + s.mockClient.EXPECT().ContainerRemove(gomock.Any(), gomock.Any()).Return(nil) +} + +func (s *ContainerEngineTestSuite) assertLaunch(eng sandboxer.Engine, expectedID string) sandboxer.Sandbox { + resp, err := eng.Launch(s.T().Context(), &sandboxer.LaunchRequest{}) + s.Require().NoError(err) + s.Require().NotNil(resp) + s.Require().NotNil(resp.Sandbox) + s.Require().NotNil(resp.JobContainer) + s.Require().Equal(expectedID, resp.JobContainer.Id) + return resp.Sandbox +} + +func (s *ContainerEngineTestSuite) TestLaunch() { + s.Run("with provisioner", func() { + img := &ocistore.Image{} + s.store.EXPECT().Image(gomock.Any(), "drassi/node:24").Return(img, nil).AnyTimes() + s.store.EXPECT().Mount(gomock.Any(), img, gomock.Any()).Return("/var/lib/drassi/node_mount", "layer-node", nil).AnyTimes() + s.store.EXPECT().Unmount(gomock.Any(), "layer-node").Return(nil).AnyTimes() + + runtimes := map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + } + + p := provision.New[*types.ContainerSpec]( + runtimes, + provision.Pull[*types.ContainerSpec](s.store), + provision.Mount[*types.ContainerSpec](s.store), + AddBindMount(), + ) + + s.mockContainerLifecycle("c-123", func(spec *types.ContainerSpec) { + s.Require().Len(spec.Mounts, 1) + s.Require().Equal("/var/lib/drassi/node_mount", spec.Mounts[0].Source) + s.Require().Equal("/opt/drassi/runtimes/node", spec.Mounts[0].Target) + }) + + eng := NewWithClient(s.mockClient, "default:image", p) + sb := s.assertLaunch(eng, "c-123") + s.Require().NoError(sb.Terminate(s.T().Context())) + }) + + s.Run("without provisioner", func() { + s.mockContainerLifecycle("c-456", func(spec *types.ContainerSpec) { + s.Require().Empty(spec.Mounts) + }) + + eng := NewWithClient(s.mockClient, "default:image", nil) + sb := s.assertLaunch(eng, "c-456") + s.Require().NoError(sb.Terminate(s.T().Context())) + }) +} + +func (s *ContainerEngineTestSuite) TestFactory() { + s.Run("with runtimes and store", func() { + f := NewFactory(DefaultConfig()) + f.SetOciStore(s.store) + f.ProvisionRuntime(map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + }) + + eng, err := f.Create() + s.Require().NoError(err) + s.Require().NotNil(eng) + _ = eng.Close() + }) + + s.Run("with runtimes but missing store returns error", func() { + f := NewFactory(DefaultConfig()) + f.ProvisionRuntime(map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + }) + + _, err := f.Create() + s.Require().Error(err) + s.Require().Contains(err.Error(), "oci store is required") + }) + + s.Run("without runtimes", func() { + f := NewFactory(DefaultConfig()) + eng, err := f.Create() + s.Require().NoError(err) + s.Require().NotNil(eng) + _ = eng.Close() + }) +} diff --git a/core/pkg/sandboxer/container/operation.go b/core/pkg/sandboxer/container/operation.go index e6a8df85..8f863ca8 100644 --- a/core/pkg/sandboxer/container/operation.go +++ b/core/pkg/sandboxer/container/operation.go @@ -7,6 +7,7 @@ package container import ( + "errors" "fmt" "path/filepath" @@ -15,20 +16,23 @@ import ( ) type addBindMountOp struct { - provision.Noop - spec *types.ContainerSpec + provision.Noop[*types.ContainerSpec] } -func AddBindMount(spec *types.ContainerSpec) provision.Operation { - return &addBindMountOp{spec: spec} +// AddBindMount returns an Operation that appends a runtime bind mount to types.ContainerSpec.Mounts. +func AddBindMount() provision.Operation[*types.ContainerSpec] { + return addBindMountOp{} } -func (op *addBindMountOp) Name() string { return "container/bind-mount" } +func (op addBindMountOp) Name() string { return "container/bind-mount" } -func (op *addBindMountOp) PreLaunch(pctx *provision.Context) error { +func (op addBindMountOp) PreLaunch(pctx *provision.Context, spec *types.ContainerSpec) (*types.ContainerSpec, error) { + if spec == nil { + return nil, errors.New("container spec cannot be nil") + } hostMountDir, ok := pctx.Get(provision.KeyHostMountDir) if !ok { - return fmt.Errorf("host mount directory not set in context") + return spec, fmt.Errorf("host mount directory not set in context") } sourcePath := hostMountDir @@ -36,36 +40,11 @@ func (op *addBindMountOp) PreLaunch(pctx *provision.Context) error { sourcePath = filepath.Join(sourcePath, pctx.Config.Subpath) } - op.spec.Mounts = append(op.spec.Mounts, &types.Mount{ - Type: "bind", - Source: sourcePath, - Target: pctx.TargetDir, + spec.Mounts = append(spec.Mounts, &types.Mount{ + Type: "bind", + Source: sourcePath, + Target: pctx.TargetDir, + ReadOnly: pctx.Config.ReadOnly, }) - return nil -} - -type addImageMountOp struct { - provision.Noop - spec *types.ContainerSpec -} - -func AddImageMount(spec *types.ContainerSpec) provision.Operation { - return &addImageMountOp{spec: spec} -} - -func (op *addImageMountOp) Name() string { return "container/image-mount" } - -func (op *addImageMountOp) PreLaunch(pctx *provision.Context) error { - mount := &types.Mount{ - Type: "image", - Source: pctx.Config.Image, - Target: pctx.TargetDir, - } - if pctx.Config.Subpath != "" { - mount.ImageOptions = &types.ImageOptions{ - Subpath: pctx.Config.Subpath, - } - } - op.spec.Mounts = append(op.spec.Mounts, mount) - return nil + return spec, nil } diff --git a/core/pkg/sandboxer/container/operation_test.go b/core/pkg/sandboxer/container/operation_test.go index 36ea5ec4..513e4f43 100644 --- a/core/pkg/sandboxer/container/operation_test.go +++ b/core/pkg/sandboxer/container/operation_test.go @@ -16,83 +16,60 @@ import ( "github.com/stretchr/testify/require" ) -func newTestSetup(t *testing.T, name string, cfg *config.Runtime) (*types.ContainerSpec, *provision.Context) { - t.Helper() - if cfg == nil { - cfg = new(config.Runtime) - } - spec := new(types.ContainerSpec) - pctx := provision.NewContext(t.Context(), name, cfg, "/opt/drassi/runtimes/"+name) - return spec, pctx -} - func TestContainerBindMount(t *testing.T) { t.Run("basic bind mount", func(t *testing.T) { - spec, pctx := newTestSetup(t, "node", nil) + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{}, "/opt/drassi/runtimes/node") pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") - op := AddBindMount(spec) + op := AddBindMount() require.Equal(t, "container/bind-mount", op.Name()) - require.NoError(t, op.PreLaunch(pctx)) + spec := new(types.ContainerSpec) + spec, err := op.PreLaunch(pctx, spec) + require.NoError(t, err) + require.Len(t, spec.Mounts, 1) + require.Equal(t, "bind", spec.Mounts[0].Type) + require.Equal(t, "/var/lib/drassi/storage/overlay/merged", spec.Mounts[0].Source) + require.Equal(t, "/opt/drassi/runtimes/node", spec.Mounts[0].Target) + require.False(t, spec.Mounts[0].ReadOnly) + }) + + t.Run("readonly bind mount", func(t *testing.T) { + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{ReadOnly: true}, "/opt/drassi/runtimes/node") + pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") + + op := AddBindMount() + spec := new(types.ContainerSpec) + spec, err := op.PreLaunch(pctx, spec) + require.NoError(t, err) require.Len(t, spec.Mounts, 1) require.Equal(t, "bind", spec.Mounts[0].Type) require.Equal(t, "/var/lib/drassi/storage/overlay/merged", spec.Mounts[0].Source) require.Equal(t, "/opt/drassi/runtimes/node", spec.Mounts[0].Target) - require.NoError(t, op.PostLaunch(pctx, nil)) + require.True(t, spec.Mounts[0].ReadOnly) }) t.Run("with subpath", func(t *testing.T) { - spec, pctx := newTestSetup(t, "python", &config.Runtime{Subpath: "opt/python"}) + pctx := provision.NewContext(t.Context(), "python", &config.Runtime{Subpath: "opt/python"}, "/opt/drassi/runtimes/python") pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") - op := AddBindMount(spec) - require.NoError(t, op.PreLaunch(pctx)) + op := AddBindMount() + spec := new(types.ContainerSpec) + spec, err := op.PreLaunch(pctx, spec) + require.NoError(t, err) require.Len(t, spec.Mounts, 1) require.Equal(t, "bind", spec.Mounts[0].Type) require.Equal(t, filepath.Join("/var/lib/drassi/storage/overlay/merged", "opt/python"), spec.Mounts[0].Source) require.Equal(t, "/opt/drassi/runtimes/python", spec.Mounts[0].Target) + require.False(t, spec.Mounts[0].ReadOnly) }) t.Run("missing host mount dir", func(t *testing.T) { - spec, pctx := newTestSetup(t, "node", nil) - - op := AddBindMount(spec) - err := op.PreLaunch(pctx) + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{}, "/opt/drassi/runtimes/node") + op := AddBindMount() + spec := new(types.ContainerSpec) + _, err := op.PreLaunch(pctx, spec) require.Error(t, err) require.Contains(t, err.Error(), "host mount directory not set in context") }) } - -func TestContainerImageMount(t *testing.T) { - t.Run("basic image mount", func(t *testing.T) { - spec, pctx := newTestSetup(t, "node", &config.Runtime{Image: "drassi/node:24"}) - - op := AddImageMount(spec) - require.Equal(t, "container/image-mount", op.Name()) - - require.NoError(t, op.PreLaunch(pctx)) - require.Len(t, spec.Mounts, 1) - require.Equal(t, "image", spec.Mounts[0].Type) - require.Equal(t, "drassi/node:24", spec.Mounts[0].Source) - require.Equal(t, "/opt/drassi/runtimes/node", spec.Mounts[0].Target) - require.Nil(t, spec.Mounts[0].ImageOptions) - require.NoError(t, op.PostLaunch(pctx, nil)) - }) - - t.Run("with subpath", func(t *testing.T) { - spec, pctx := newTestSetup(t, "python", &config.Runtime{ - Image: "drassi/python:3.12", - Subpath: "opt/python", - }) - - op := AddImageMount(spec) - require.NoError(t, op.PreLaunch(pctx)) - require.Len(t, spec.Mounts, 1) - require.Equal(t, "image", spec.Mounts[0].Type) - require.Equal(t, "drassi/python:3.12", spec.Mounts[0].Source) - require.Equal(t, "/opt/drassi/runtimes/python", spec.Mounts[0].Target) - require.NotNil(t, spec.Mounts[0].ImageOptions) - require.Equal(t, "opt/python", spec.Mounts[0].ImageOptions.Subpath) - }) -} diff --git a/core/pkg/sandboxer/container/sandbox.go b/core/pkg/sandboxer/container/sandbox.go index 81bb6f72..835aeb5e 100644 --- a/core/pkg/sandboxer/container/sandbox.go +++ b/core/pkg/sandboxer/container/sandbox.go @@ -30,6 +30,7 @@ var defaultLayout = sandboxer.Layout{ Temp: filepath.Join(jobDir, "temp"), Actions: filepath.Join(jobDir, "actions"), Tools: filepath.Join(jobDir, "tools"), + Runtimes: filepath.Join(jobDir, "runtimes"), } type sandbox struct { diff --git a/core/pkg/sandboxer/factory.go b/core/pkg/sandboxer/factory.go index 2c10ecd8..cb6a8f32 100644 --- a/core/pkg/sandboxer/factory.go +++ b/core/pkg/sandboxer/factory.go @@ -20,7 +20,8 @@ import ( type Factory interface { // SupportContainer(config) // TODO - ProvisionRuntime(store ocistore.Manager, config map[string]*config.Runtime) + SetOciStore(store ocistore.Manager) + ProvisionRuntime(config map[string]*config.Runtime) Create() (Engine, error) } diff --git a/core/pkg/sandboxer/host/engine.go b/core/pkg/sandboxer/host/engine.go index 9f849cbc..656542ff 100644 --- a/core/pkg/sandboxer/host/engine.go +++ b/core/pkg/sandboxer/host/engine.go @@ -8,6 +8,7 @@ package host import ( "context" + "errors" "net/url" "os" "path/filepath" @@ -17,12 +18,13 @@ import ( "drassi.run/core/config" c "drassi.run/core/pkg/container" "drassi.run/core/pkg/container/docker" + "drassi.run/core/pkg/runtime/provision" "drassi.run/core/pkg/sandboxer" "drassi.run/core/pkg/sandboxer/container" - "drassi.run/core/pkg/store/oci" - "drassi.run/core/util/fs" - "drassi.run/core/util/path" - "drassi.run/core/util/string" + ocistore "drassi.run/core/pkg/store/oci" + xfs "drassi.run/core/util/fs" + xpath "drassi.run/core/util/path" + xstring "drassi.run/core/util/string" ) func init() { @@ -50,8 +52,11 @@ type factory struct { runtimes map[string]*config.Runtime } -func (f *factory) ProvisionRuntime(store ocistore.Manager, config map[string]*config.Runtime) { +func (f *factory) SetOciStore(store ocistore.Manager) { f.store = store +} + +func (f *factory) ProvisionRuntime(config map[string]*config.Runtime) { f.runtimes = config } @@ -60,7 +65,19 @@ func (f *factory) Create() (sandboxer.Engine, error) { } func (f *factory) doCreate() (sandboxer.Engine, error) { - return New(f.cfg) + var prov *provision.Provisioner[string] + if len(f.runtimes) > 0 { + if f.store == nil { + return nil, errors.New("oci store is required when runtimes are configured") + } + prov = provision.New[string]( + f.runtimes, + provision.Pull[string](f.store), + provision.Mount[string](f.store), + Symlink[string](), + ) + } + return New(f.cfg, prov) } type Config struct { @@ -70,9 +87,10 @@ type Config struct { type engine struct { Config + provisioner *provision.Provisioner[string] } -func New(config *Config) (sandboxer.Engine, error) { +func New(config *Config, prov *provision.Provisioner[string]) (sandboxer.Engine, error) { if d, err := xpath.ResolveDir(config.RootDir); err != nil { return nil, err } else { @@ -83,31 +101,55 @@ func New(config *Config) (sandboxer.Engine, error) { return nil, err } - return &engine{Config: *config}, nil -} - -func (e *engine) Close() error { - return nil + return &engine{ + Config: *config, + provisioner: prov, + }, nil } func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*sandboxer.LaunchResponse, error) { sandboxDir := e.sandboxDir(req) sandboxDir = filepath.Join(e.RootDir, sandboxDir) - sb, err := newSandbox(sandboxDir) + launcher := e.launch + if prov := e.provisioner; prov != nil { + runtimeDir := filepath.Join(sandboxDir, "runtime") + launcher = prov.Launch(runtimeDir, launcher) + } + sb, err := launcher(ctx, sandboxDir) if err != nil { return nil, err } - sb.layout.Runtimes = e.RuntimeDir client, err := docker.New() if err != nil { + _ = sb.Terminate(ctx) return nil, err } client = c.WithTelemetry(client) b := container.NewBootstrapper(client) - return b.Bootstrap(ctx, sb, req) + resp, err := b.Bootstrap(ctx, sb, req) + if err != nil { + _ = sb.Terminate(ctx) + return nil, err + } + return resp, nil +} + +func (e *engine) launch(ctx context.Context, sandboxDir string) (sandboxer.Sandbox, error) { + sb, err := newSandbox(sandboxDir) + if err != nil { + return nil, err + } + sb.layout.Runtimes = e.RuntimeDir + if sb.layout.Runtimes != "" { + if err := os.MkdirAll(sb.layout.Runtimes, xfs.DirPerm); err != nil { + _ = sb.Terminate(ctx) + return nil, err + } + } + return sb, nil } func (e *engine) sandboxDir(req *sandboxer.LaunchRequest) string { @@ -129,3 +171,7 @@ func (e *engine) sandboxDir(req *sandboxer.LaunchRequest) string { path := filepath.Join(server, repo, workflow, job, run+"_"+attempt) return path } + +func (e *engine) Close() error { + return nil +} diff --git a/core/pkg/sandboxer/host/engine_test.go b/core/pkg/sandboxer/host/engine_test.go new file mode 100644 index 00000000..4e0da58a --- /dev/null +++ b/core/pkg/sandboxer/host/engine_test.go @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package host + +import ( + "os" + "path/filepath" + "testing" + + "drassi.run/core/config" + mock_store "drassi.run/core/mock/store/oci" + "drassi.run/core/pkg/model/records" + "drassi.run/core/pkg/runtime/provision" + "drassi.run/core/pkg/sandboxer" + ocistore "drassi.run/core/pkg/store/oci" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +func TestHostEngineSuite(t *testing.T) { + suite.Run(t, new(HostEngineTestSuite)) +} + +type HostEngineTestSuite struct { + suite.Suite + ctrl *gomock.Controller + store *mock_store.MockManager + tempDir string + runtimeDir string + cfg *Config +} + +func (s *HostEngineTestSuite) SetupTest() { + s.ctrl = gomock.NewController(s.T()) + s.store = mock_store.NewMockManager(s.ctrl) + s.tempDir = s.T().TempDir() + s.runtimeDir = filepath.Join(s.tempDir, "opt_drassi_runtimes") + s.cfg = &Config{ + RootDir: s.tempDir, + RuntimeDir: s.runtimeDir, + } +} + +func (s *HostEngineTestSuite) assertLaunch(eng sandboxer.Engine) sandboxer.Sandbox { + req := &sandboxer.LaunchRequest{ + Forge: &records.Forge{ + Repository: "drassi/test", + Workflow: "build.yml", + Job: "test", + RunId: "1", + RunAttempt: "1", + }, + } + + resp, err := eng.Launch(s.T().Context(), req) + s.Require().NoError(err) + s.Require().NotNil(resp) + s.Require().NotNil(resp.Sandbox) + return resp.Sandbox +} + +func (s *HostEngineTestSuite) TestLaunch() { + s.Run("with provisioner", func() { + mountDir := filepath.Join(s.tempDir, "node_mount") + s.Require().NoError(os.MkdirAll(mountDir, 0755)) + + img := &ocistore.Image{} + s.store.EXPECT().Image(gomock.Any(), "drassi/node:24").Return(img, nil).Times(1) + s.store.EXPECT().Mount(gomock.Any(), img, gomock.Any()).Return(mountDir, "layer-node", nil).Times(1) + s.store.EXPECT().Unmount(gomock.Any(), "layer-node").Return(nil).Times(1) + + runtimes := map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + } + + p := provision.New[string]( + runtimes, + provision.Pull[string](s.store), + provision.Mount[string](s.store), + Symlink[string](), + ) + + eng, err := New(s.cfg, p) + s.Require().NoError(err) + + sb := s.assertLaunch(eng) + + // Symlink is created + symlinkPath := filepath.Join(s.runtimeDir, "node") + target, err := os.Readlink(symlinkPath) + s.Require().NoError(err) + s.Require().Equal(mountDir, target) + + // Terminate sandbox unmounts layers and removes workspace + s.Require().NoError(sb.Terminate(s.T().Context())) + }) + + s.Run("without provisioner", func() { + eng, err := New(s.cfg, nil) + s.Require().NoError(err) + + sb := s.assertLaunch(eng) + s.Require().NoError(sb.Terminate(s.T().Context())) + }) +} + + +func (s *HostEngineTestSuite) TestFactory() { + s.Run("with runtimes and store", func() { + cfg := DefaultConfig() + cfg.RootDir = s.T().TempDir() + cfg.RuntimeDir = filepath.Join(cfg.RootDir, "runtimes") + f := NewFactory(cfg) + f.SetOciStore(s.store) + f.ProvisionRuntime(map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + }) + eng, err := f.Create() + s.Require().NoError(err) + s.Require().NotNil(eng) + _ = eng.Close() + }) + + s.Run("with runtimes but missing store returns error", func() { + cfg := DefaultConfig() + cfg.RootDir = s.T().TempDir() + cfg.RuntimeDir = filepath.Join(cfg.RootDir, "runtimes") + f := NewFactory(cfg) + f.ProvisionRuntime(map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + }) + _, err := f.Create() + s.Require().Error(err) + s.Require().Contains(err.Error(), "oci store is required") + }) + + s.Run("without runtimes", func() { + cfg := DefaultConfig() + cfg.RootDir = s.T().TempDir() + cfg.RuntimeDir = filepath.Join(cfg.RootDir, "runtimes") + f := NewFactory(cfg) + eng, err := f.Create() + s.Require().NoError(err) + s.Require().NotNil(eng) + _ = eng.Close() + }) +} + +func (s *HostEngineTestSuite) TestNew() { + s.Run("without panic when nil", func() { + cfg := DefaultConfig() + cfg.RootDir = s.T().TempDir() + eng, err := New(cfg, nil) + s.Require().NoError(err) + s.Require().NotNil(eng) + _ = eng.Close() + }) +} diff --git a/core/pkg/sandboxer/host/operation.go b/core/pkg/sandboxer/host/operation.go index ebdf128d..696b0c25 100644 --- a/core/pkg/sandboxer/host/operation.go +++ b/core/pkg/sandboxer/host/operation.go @@ -7,6 +7,7 @@ package host import ( + "context" "fmt" "os" "path/filepath" @@ -15,22 +16,22 @@ import ( "drassi.run/core/pkg/sandboxer" ) -type symlinkOp struct { - provision.Noop +type symlinkOp[Req any] struct { + provision.Noop[Req] } // Symlink returns a provision.Operation that symlinks the runtime directory in the sandbox // to the host mount directory. -func Symlink() provision.Operation { - return symlinkOp{} +func Symlink[Req any]() provision.Operation[Req] { + return symlinkOp[Req]{} } -func (op symlinkOp) Name() string { return "host/symlink" } +func (op symlinkOp[Req]) Name() string { return "host/symlink" } -func (op symlinkOp) PostLaunch(pctx *provision.Context, sb sandboxer.Sandbox) error { +func (op symlinkOp[Req]) PostLaunch(pctx *provision.Context, sb sandboxer.Sandbox) (sandboxer.Sandbox, error) { hostMountDir, ok := pctx.Get(provision.KeyHostMountDir) if !ok { - return fmt.Errorf("host mount directory not set in context") + return sb, fmt.Errorf("host mount directory not set in context") } runtimeDir := sb.Layout().Runtimes @@ -40,5 +41,11 @@ func (op symlinkOp) PostLaunch(pctx *provision.Context, sb sandboxer.Sandbox) er } _ = os.Remove(target) // Remove existing symlink if present - return os.Symlink(hostMountDir, target) + if err := os.Symlink(hostMountDir, target); err != nil { + return sb, err + } + sb = sandboxer.AddBeforeCleanup(sb, func(context.Context) error { + return os.Remove(target) + }) + return sb, nil } diff --git a/core/pkg/sandboxer/host/operation_test.go b/core/pkg/sandboxer/host/operation_test.go index 95b731fa..e9bb2e95 100644 --- a/core/pkg/sandboxer/host/operation_test.go +++ b/core/pkg/sandboxer/host/operation_test.go @@ -19,47 +19,48 @@ import ( "go.uber.org/mock/gomock" ) -func newTestSetup(t *testing.T, name string, cfg *config.Runtime) (provision.Operation, *provision.Context, *mock_sandboxer.MockSandbox, string) { - t.Helper() - if cfg == nil { - cfg = new(config.Runtime) - } +func TestHostSymlink(t *testing.T) { + ctrl := gomock.NewController(t) runtimesDir := filepath.Join(t.TempDir(), "runtimes") require.NoError(t, os.MkdirAll(runtimesDir, 0755)) - pctx := provision.NewContext(t.Context(), name, cfg, filepath.Join(runtimesDir, name)) - op := Symlink() - sb := mock_sandboxer.NewMockSandbox(gomock.NewController(t)) + op := Symlink[any]() + require.Equal(t, "host/symlink", op.Name()) + + sb := mock_sandboxer.NewMockSandbox(ctrl) sb.EXPECT().Layout().Return(&sandboxer.Layout{Runtimes: runtimesDir}).AnyTimes() - return op, pctx, sb, runtimesDir -} -func TestHostSymlink(t *testing.T) { t.Run("basic symlink", func(t *testing.T) { - op, pctx, sb, runtimesDir := newTestSetup(t, "node", nil) - require.Equal(t, "host/symlink", op.Name()) - require.NoError(t, op.PreLaunch(pctx)) - mountDir := filepath.Join(t.TempDir(), "mount_merged") require.NoError(t, os.MkdirAll(mountDir, 0755)) + + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{}, filepath.Join(runtimesDir, "node")) pctx.Set(provision.KeyHostMountDir, mountDir) - require.NoError(t, op.PostLaunch(pctx, sb)) + resSb, err := op.PostLaunch(pctx, sb) + require.NoError(t, err) + require.Equal(t, sb, sandboxer.Unwrap(resSb)) target := filepath.Join(runtimesDir, "node") targetInfo, err := os.Lstat(target) require.NoError(t, err) require.True(t, targetInfo.Mode()&os.ModeSymlink != 0) + + sb.EXPECT().Terminate(gomock.Any()).Return(nil) + require.NoError(t, resSb.Terminate(t.Context())) + _, err = os.Lstat(target) + require.True(t, os.IsNotExist(err)) }) t.Run("with subpath", func(t *testing.T) { - op, pctx, sb, runtimesDir := newTestSetup(t, "custom", &config.Runtime{Subpath: "custom/sub"}) - subMountDir := filepath.Join(t.TempDir(), "sub_merged") require.NoError(t, os.MkdirAll(filepath.Join(subMountDir, "custom/sub"), 0755)) + + pctx := provision.NewContext(t.Context(), "custom", &config.Runtime{Subpath: "custom/sub"}, filepath.Join(runtimesDir, "custom")) pctx.Set(provision.KeyHostMountDir, subMountDir) - require.NoError(t, op.PostLaunch(pctx, sb)) + _, err := op.PostLaunch(pctx, sb) + require.NoError(t, err) targetSub := filepath.Join(runtimesDir, "custom") linkTarget, err := os.Readlink(targetSub) @@ -68,10 +69,9 @@ func TestHostSymlink(t *testing.T) { }) t.Run("missing host mount dir", func(t *testing.T) { - op, pctx, sb, _ := newTestSetup(t, "missing", nil) - - err := op.PostLaunch(pctx, sb) + pctx := provision.NewContext(t.Context(), "missing", &config.Runtime{}, filepath.Join(runtimesDir, "missing")) + _, err := op.PostLaunch(pctx, sb) require.Error(t, err) - require.ErrorContains(t, err, "host mount directory not set in context") + require.Contains(t, err.Error(), "host mount directory not set in context") }) } diff --git a/core/pkg/sandboxer/incus/engine.go b/core/pkg/sandboxer/incus/engine.go index e4a61296..bdd9570a 100644 --- a/core/pkg/sandboxer/incus/engine.go +++ b/core/pkg/sandboxer/incus/engine.go @@ -8,7 +8,11 @@ package incus import ( "context" + "errors" + "maps" + "net" "path" + "slices" "strings" "sync" @@ -16,6 +20,7 @@ import ( c "drassi.run/core/pkg/container" "drassi.run/core/pkg/container/docker" "drassi.run/core/pkg/model/records" + "drassi.run/core/pkg/runtime/provision" "drassi.run/core/pkg/sandboxer" "drassi.run/core/pkg/sandboxer/container" "drassi.run/core/pkg/store/oci" @@ -53,8 +58,11 @@ type factory struct { runtimes map[string]*config.Runtime } -func (f *factory) ProvisionRuntime(store ocistore.Manager, config map[string]*config.Runtime) { +func (f *factory) SetOciStore(store ocistore.Manager) { f.store = store +} + +func (f *factory) ProvisionRuntime(config map[string]*config.Runtime) { f.runtimes = config } @@ -63,7 +71,19 @@ func (f *factory) Create() (sandboxer.Engine, error) { } func (f *factory) doCreate() (sandboxer.Engine, error) { - return New(f.cfg) + var prov *provision.Provisioner[*Template] + if len(f.runtimes) > 0 { + if f.store == nil { + return nil, errors.New("oci store is required when runtimes are configured") + } + prov = provision.New[*Template]( + f.runtimes, + provision.Pull[*Template](f.store), + provision.Mount[*Template](f.store), + AddDiskDevice(), + ) + } + return New(f.cfg, prov) } type Config struct { @@ -74,6 +94,8 @@ type Config struct { // Template for create incus VM // [github.com/lxc/incus/v6/shared/api.InstancesPost] type Template struct { + Name string `toml:"name,omitempty" json:"name,omitempty"` + // OCI image name, e.g: ghcr.io/drassi-run/ubuntu:22.04 Image string `toml:"source" json:"image"` @@ -101,46 +123,102 @@ type Template struct { Ephemeral bool `toml:"ephemeral" json:"ephemeral,omitempty"` } +func (t *Template) Clone() *Template { + if t == nil { + return nil + } + c := *t + if t.Profiles != nil { + c.Profiles = slices.Clone(t.Profiles) + } + if t.Config != nil { + c.Config = maps.Clone(t.Config) + } + if t.Devices != nil { + c.Devices = make(map[string]map[string]string, len(t.Devices)) + for k, v := range t.Devices { + c.Devices[k] = maps.Clone(v) + } + } + return &c +} + type engine struct { - client incusclient.InstanceServer - template *Template - source *incusapi.InstanceSource + client incusclient.InstanceServer + template *Template + source *incusapi.InstanceSource + provisioner *provision.Provisioner[*Template] } -func New(config *Config) (sandboxer.Engine, error) { +func New(config *Config, prov *provision.Provisioner[*Template]) (sandboxer.Engine, error) { if client, err := incusclient.ConnectIncusUnix(config.Endpoint, nil); err != nil { return nil, err } else if source, err := instanceSource(config.Template.Image); err != nil { return nil, err } else { e := &engine{ - client: client, - template: &config.Template, - source: source, + client: client, + template: &config.Template, + source: source, + provisioner: prov, } return e, nil } } -func (e *engine) Close() error { - e.client.Disconnect() - return nil +func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*sandboxer.LaunchResponse, error) { + tmpl := e.template.Clone() + tmpl.Name = e.sandboxName(req.Forge) + + launcher := e.launch + if prov := e.provisioner; prov != nil { + launcher = prov.Launch(defaultRuntimeDir, launcher) + } + sb, err := launcher(ctx, tmpl) + if err != nil { + return nil, err + } + + d, ok := sandboxer.Unwrap(sb).(interface { + Dialer(cmd []string) func(ctx context.Context, network, addr string) (net.Conn, error) + }) + if !ok { + _ = sb.Terminate(ctx) + return nil, errors.New("sandbox does not support dialer") + } + dialer := d.Dialer(docker.ProxyCommand("")) + client, err := docker.New(dockerclient.WithDialContext(dialer)) + if err != nil { + _ = sb.Terminate(ctx) + return nil, err + } + client = c.WithTelemetry(client) + + s := sandboxer.AddBeforeCleanup(sb, func(context.Context) error { + return client.Close() + }) + b := container.NewBootstrapper(client) + resp, err := b.Bootstrap(ctx, s, req) + if err != nil { + _ = s.Terminate(ctx) + return nil, err + } + return resp, nil } -func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*sandboxer.LaunchResponse, error) { - name := e.sandboxName(req.Forge) +func (e *engine) launch(ctx context.Context, tmpl *Template) (sandboxer.Sandbox, error) { iReq := incusapi.InstancesPost{ - Name: name, + Name: tmpl.Name, Start: true, Source: *e.source, Type: incusapi.InstanceTypeContainer, - InstanceType: e.template.InstanceSize, + InstanceType: tmpl.InstanceSize, InstancePut: incusapi.InstancePut{ - Architecture: e.template.Architecture, - Config: e.template.Config, - Devices: e.template.Devices, - Ephemeral: e.template.Ephemeral, - Profiles: e.template.Profiles, + Architecture: tmpl.Architecture, + Config: tmpl.Config, + Devices: tmpl.Devices, + Ephemeral: tmpl.Ephemeral, + Profiles: tmpl.Profiles, }, } if op, err := e.client.CreateInstance(iReq); err != nil { @@ -149,23 +227,11 @@ func (e *engine) Launch(ctx context.Context, req *sandboxer.LaunchRequest) (*san return nil, err } - sb, err := newSandbox(e.client, name) + sb, err := newSandbox(e.client, tmpl.Name) if err != nil { return nil, err } - - dialer := sb.Dialer(docker.ProxyCommand("")) - client, err := docker.New(dockerclient.WithDialContext(dialer)) - if err != nil { - return nil, err - } - client = c.WithTelemetry(client) - - s := sandboxer.AddBeforeCleanup(sb, func(context.Context) error { - return client.Close() - }) - b := container.NewBootstrapper(client) - return b.Bootstrap(ctx, s, req) + return sb, nil } func (e *engine) sandboxName(forge *records.Forge) string { @@ -203,3 +269,8 @@ func instanceSource(uri string) (*incusapi.InstanceSource, error) { } return source, nil } + +func (e *engine) Close() error { + e.client.Disconnect() + return nil +} diff --git a/core/pkg/sandboxer/incus/engine_test.go b/core/pkg/sandboxer/incus/engine_test.go new file mode 100644 index 00000000..fe3e2bf7 --- /dev/null +++ b/core/pkg/sandboxer/incus/engine_test.go @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: (c) 2024 The Drassi Authors + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package incus + +import ( + "context" + "testing" + + "drassi.run/core/config" + mock_sandboxer "drassi.run/core/mock/sandboxer" + mock_store "drassi.run/core/mock/store/oci" + "drassi.run/core/pkg/runtime/provision" + "drassi.run/core/pkg/sandboxer" + ocistore "drassi.run/core/pkg/store/oci" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +func TestIncusEngineSuite(t *testing.T) { + suite.Run(t, new(IncusEngineTestSuite)) +} + +type IncusEngineTestSuite struct { + suite.Suite + ctrl *gomock.Controller + store *mock_store.MockManager +} + +func (s *IncusEngineTestSuite) SetupTest() { + s.ctrl = gomock.NewController(s.T()) + s.store = mock_store.NewMockManager(s.ctrl) +} + +func (s *IncusEngineTestSuite) TestProvisionerIntegration() { + img := &ocistore.Image{} + s.store.EXPECT().Image(gomock.Any(), "drassi/node:24").Return(img, nil).Times(1) + s.store.EXPECT().Mount(gomock.Any(), img, gomock.Any()).Return("/var/lib/drassi/node_mount", "layer-node", nil).Times(1) + s.store.EXPECT().Unmount(gomock.Any(), "layer-node").Return(nil).Times(1) + + runtimes := map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + } + + p := provision.New[*Template]( + runtimes, + provision.Pull[*Template](s.store), + provision.Mount[*Template](s.store), + AddDiskDevice(), + ) + + tmpl := &Template{} + launched := false + mockSb := mock_sandboxer.NewMockSandbox(s.ctrl) + mockSb.EXPECT().Terminate(gomock.Any()).Return(nil).Times(1) + + launcher := func(ctx context.Context, tmpl *Template) (sandboxer.Sandbox, error) { + launched = true + s.Require().Contains(tmpl.Devices, "runtime-node") + s.Require().Equal("disk", tmpl.Devices["runtime-node"]["type"]) + s.Require().Equal("/var/lib/drassi/node_mount", tmpl.Devices["runtime-node"]["source"]) + s.Require().Equal("/opt/drassi/runtimes/node", tmpl.Devices["runtime-node"]["path"]) + return mockSb, nil + } + + sb, err := p.Launch("/opt/drassi/runtimes", launcher)(s.T().Context(), tmpl) + s.Require().NoError(err) + s.Require().True(launched) + s.Require().NotNil(sb) + + // Verify unmount cleanup runs on terminate + s.Require().NoError(sb.Terminate(s.T().Context())) +} + +func (s *IncusEngineTestSuite) TestTemplateClone() { + s.Run("nil template", func() { + var tmpl *Template + s.Require().Nil(tmpl.Clone()) + }) + + s.Run("deep copy fields", func() { + orig := &Template{ + Name: "test-instance", + Image: "ubuntu:22.04", + Architecture: "x86_64", + InstanceSize: "t1.micro", + Profiles: []string{"default", "custom"}, + Config: map[string]string{ + "security.nesting": "true", + }, + Devices: map[string]map[string]string{ + "root": { + "type": "disk", + "path": "/", + }, + }, + Ephemeral: true, + } + + clone := orig.Clone() + s.Require().NotNil(clone) + s.Require().Equal(orig.Name, clone.Name) + s.Require().Equal(orig.Image, clone.Image) + s.Require().Equal(orig.Architecture, clone.Architecture) + s.Require().Equal(orig.InstanceSize, clone.InstanceSize) + s.Require().Equal(orig.Ephemeral, clone.Ephemeral) + s.Require().Equal(orig.Profiles, clone.Profiles) + s.Require().Equal(orig.Config, clone.Config) + s.Require().Equal(orig.Devices, clone.Devices) + + // Verify Profiles is deep copied + clone.Profiles[0] = "modified" + s.Require().Equal("default", orig.Profiles[0]) + + // Verify Config is deep copied + clone.Config["security.nesting"] = "false" + clone.Config["new.key"] = "val" + s.Require().Equal("true", orig.Config["security.nesting"]) + s.Require().NotContains(orig.Config, "new.key") + + // Verify Devices outer map is deep copied + clone.Devices["extra"] = map[string]string{"type": "nic"} + s.Require().NotContains(orig.Devices, "extra") + + // Verify Devices inner map is deep copied + clone.Devices["root"]["path"] = "/mnt" + s.Require().Equal("/", orig.Devices["root"]["path"]) + }) +} + +func (s *IncusEngineTestSuite) TestFactory() { + s.Run("with runtimes and store", func() { + f := NewFactory(DefaultConfig()) + f.SetOciStore(s.store) + f.ProvisionRuntime(map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + }) + s.Require().NotPanics(func() { + _, _ = f.Create() + }) + }) + + s.Run("with runtimes but missing store returns error", func() { + f := NewFactory(DefaultConfig()) + f.ProvisionRuntime(map[string]*config.Runtime{ + "node": {Image: "drassi/node:24"}, + }) + _, err := f.Create() + s.Require().Error(err) + s.Require().Contains(err.Error(), "oci store is required") + }) + + s.Run("without runtimes", func() { + f := NewFactory(DefaultConfig()) + s.Require().NotPanics(func() { + _, _ = f.Create() + }) + }) +} + +func (s *IncusEngineTestSuite) TestNew() { + s.Run("without panic when nil", func() { + cfg := DefaultConfig() + s.Require().NotPanics(func() { + _, _ = New(cfg, nil) + }) + }) +} diff --git a/core/pkg/sandboxer/incus/operation.go b/core/pkg/sandboxer/incus/operation.go index 3711af36..0d706684 100644 --- a/core/pkg/sandboxer/incus/operation.go +++ b/core/pkg/sandboxer/incus/operation.go @@ -7,27 +7,32 @@ package incus import ( + "errors" "fmt" "path/filepath" + "strconv" "drassi.run/core/pkg/runtime/provision" ) type addDiskDeviceOp struct { - provision.Noop - template *Template + provision.Noop[*Template] } -func AddDiskDevice(template *Template) provision.Operation { - return &addDiskDeviceOp{template: template} +// AddDiskDevice returns an Operation that injects a runtime disk device into Template.Devices. +func AddDiskDevice() provision.Operation[*Template] { + return addDiskDeviceOp{} } -func (op *addDiskDeviceOp) Name() string { return "incus/disk-device" } +func (op addDiskDeviceOp) Name() string { return "incus/disk-device" } -func (op *addDiskDeviceOp) PreLaunch(pctx *provision.Context) error { +func (op addDiskDeviceOp) PreLaunch(pctx *provision.Context, tmpl *Template) (*Template, error) { + if tmpl == nil { + return nil, errors.New("template cannot be nil") + } hostMountDir, ok := pctx.Get(provision.KeyHostMountDir) if !ok { - return fmt.Errorf("host mount directory not set in context") + return tmpl, fmt.Errorf("host mount directory not set in context") } sourcePath := hostMountDir @@ -35,13 +40,14 @@ func (op *addDiskDeviceOp) PreLaunch(pctx *provision.Context) error { sourcePath = filepath.Join(sourcePath, pctx.Config.Subpath) } - if op.template.Devices == nil { - op.template.Devices = make(map[string]map[string]string) + if tmpl.Devices == nil { + tmpl.Devices = make(map[string]map[string]string) } - op.template.Devices["runtime-"+pctx.RuntimeName] = map[string]string{ - "type": "disk", - "source": sourcePath, - "path": pctx.TargetDir, + tmpl.Devices["runtime-"+pctx.RuntimeName] = map[string]string{ + "type": "disk", + "source": sourcePath, + "path": pctx.TargetDir, + "readonly": strconv.FormatBool(pctx.Config.ReadOnly), } - return nil + return tmpl, nil } diff --git a/core/pkg/sandboxer/incus/operation_test.go b/core/pkg/sandboxer/incus/operation_test.go index 8694c72b..48526c4e 100644 --- a/core/pkg/sandboxer/incus/operation_test.go +++ b/core/pkg/sandboxer/incus/operation_test.go @@ -15,51 +15,62 @@ import ( "github.com/stretchr/testify/require" ) -func newTestSetup(t *testing.T, name string, cfg *config.Runtime) (*Template, *provision.Context) { - t.Helper() - if cfg == nil { - cfg = new(config.Runtime) - } - tmpl := new(Template) - pctx := provision.NewContext(t.Context(), name, cfg, "/opt/drassi/runtimes/"+name) - return tmpl, pctx -} - func TestIncusDiskDevice(t *testing.T) { t.Run("basic disk device", func(t *testing.T) { - tmpl, pctx := newTestSetup(t, "node", nil) + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{}, "/opt/drassi/runtimes/node") pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") - op := AddDiskDevice(tmpl) + op := AddDiskDevice() require.Equal(t, "incus/disk-device", op.Name()) - require.NoError(t, op.PreLaunch(pctx)) + tmpl := new(Template) + tmpl, err := op.PreLaunch(pctx, tmpl) + require.NoError(t, err) + require.Contains(t, tmpl.Devices, "runtime-node") + dev := tmpl.Devices["runtime-node"] + require.Equal(t, "disk", dev["type"]) + require.Equal(t, "/var/lib/drassi/storage/overlay/merged", dev["source"]) + require.Equal(t, "/opt/drassi/runtimes/node", dev["path"]) + require.Equal(t, "false", dev["readonly"]) + }) + + t.Run("readonly disk device", func(t *testing.T) { + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{ReadOnly: true}, "/opt/drassi/runtimes/node") + pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") + + op := AddDiskDevice() + tmpl := new(Template) + tmpl, err := op.PreLaunch(pctx, tmpl) + require.NoError(t, err) require.Contains(t, tmpl.Devices, "runtime-node") dev := tmpl.Devices["runtime-node"] require.Equal(t, "disk", dev["type"]) require.Equal(t, "/var/lib/drassi/storage/overlay/merged", dev["source"]) require.Equal(t, "/opt/drassi/runtimes/node", dev["path"]) - require.NoError(t, op.PostLaunch(pctx, nil)) + require.Equal(t, "true", dev["readonly"]) }) t.Run("with subpath", func(t *testing.T) { - tmpl, pctx := newTestSetup(t, "python", &config.Runtime{Subpath: "opt/python"}) + pctx := provision.NewContext(t.Context(), "python", &config.Runtime{Subpath: "opt/python"}, "/opt/drassi/runtimes/python") pctx.Set(provision.KeyHostMountDir, "/var/lib/drassi/storage/overlay/merged") - op := AddDiskDevice(tmpl) - require.NoError(t, op.PreLaunch(pctx)) + op := AddDiskDevice() + tmpl := new(Template) + tmpl, err := op.PreLaunch(pctx, tmpl) + require.NoError(t, err) require.Contains(t, tmpl.Devices, "runtime-python") dev := tmpl.Devices["runtime-python"] require.Equal(t, "disk", dev["type"]) require.Equal(t, filepath.Join("/var/lib/drassi/storage/overlay/merged", "opt/python"), dev["source"]) require.Equal(t, "/opt/drassi/runtimes/python", dev["path"]) + require.Equal(t, "false", dev["readonly"]) }) t.Run("missing host mount dir", func(t *testing.T) { - tmpl, pctx := newTestSetup(t, "node", nil) - - op := AddDiskDevice(tmpl) - err := op.PreLaunch(pctx) + pctx := provision.NewContext(t.Context(), "node", &config.Runtime{}, "/opt/drassi/runtimes/node") + op := AddDiskDevice() + tmpl := new(Template) + _, err := op.PreLaunch(pctx, tmpl) require.Error(t, err) require.Contains(t, err.Error(), "host mount directory not set in context") }) diff --git a/core/pkg/sandboxer/incus/sandbox.go b/core/pkg/sandboxer/incus/sandbox.go index 108489e7..7410f547 100644 --- a/core/pkg/sandboxer/incus/sandbox.go +++ b/core/pkg/sandboxer/incus/sandbox.go @@ -38,6 +38,8 @@ type sandbox struct { uid, gid uint32 } +const defaultRuntimeDir = "/opt/drassi/runtimes" + func newSandbox(client incusclient.InstanceServer, inst string) (*sandbox, error) { dir := "/opt/drassi/" sb := &sandbox{ @@ -48,6 +50,7 @@ func newSandbox(client incusclient.InstanceServer, inst string) (*sandbox, error Temp: filepath.Join(dir, "temp"), Actions: filepath.Join(dir, "actions"), Tools: filepath.Join(dir, "tools"), + Runtimes: defaultRuntimeDir, }, } diff --git a/core/pkg/sandboxer/sandbox_decorator.go b/core/pkg/sandboxer/sandbox_decorator.go index 7325bea4..26ddfa7b 100644 --- a/core/pkg/sandboxer/sandbox_decorator.go +++ b/core/pkg/sandboxer/sandbox_decorator.go @@ -68,3 +68,21 @@ func AddAfterCleanup(sb Sandbox, fns ...Cleanup) Sandbox { afterCleanup: fns, } } + +func (s *decoratedSandbox) Unwrap() Sandbox { + if s == nil { + return nil + } + return s.Sandbox +} + +// Unwrap recursively unwraps decorated sandbox wrappers until reaching the base Sandbox. +func Unwrap(sb Sandbox) Sandbox { + for { + if u, ok := sb.(interface{ Unwrap() Sandbox }); ok { + sb = u.Unwrap() + } else { + return sb + } + } +} diff --git a/runner-gha/cmd/launch/launch.go b/runner-gha/cmd/launch/launch.go index 4e24a3d7..9c389622 100644 --- a/runner-gha/cmd/launch/launch.go +++ b/runner-gha/cmd/launch/launch.go @@ -110,7 +110,8 @@ func (l *launcher) Init(ctx context.Context, opts *options) (err error) { } else if factory, err := sandboxer.NewFactory(sbConfig); err != nil { return err } else { - factory.ProvisionRuntime(l.ociStore, cfg.Runtimes) + factory.SetOciStore(l.ociStore) + factory.ProvisionRuntime(cfg.Runtimes) if sb, err := factory.Create(); err != nil { return err } else { diff --git a/runner-gha/cmd/migrate/migrate_test.go b/runner-gha/cmd/migrate/migrate_test.go index 6979405e..b6c18018 100644 --- a/runner-gha/cmd/migrate/migrate_test.go +++ b/runner-gha/cmd/migrate/migrate_test.go @@ -101,7 +101,7 @@ func TestMigrateCommand(t *testing.T) { require.True(t, ok) factory, err := sandboxer.NewFactory(sb) require.NoError(t, err) - factory.ProvisionRuntime(nil, cfg.Runtimes) + factory.ProvisionRuntime(cfg.Runtimes) engine, err := factory.Create() require.NoError(t, err) require.NotNil(t, engine) diff --git a/runner-gitea/cmd/launch/launch.go b/runner-gitea/cmd/launch/launch.go index 6f88f58b..6663aeef 100644 --- a/runner-gitea/cmd/launch/launch.go +++ b/runner-gitea/cmd/launch/launch.go @@ -209,7 +209,8 @@ func (c *launcher) loadSandboxer(config *giteaconfig.Config, name string) error } else if factory, err := sandboxer.NewFactory(sbConfig); err != nil { return err } else { - factory.ProvisionRuntime(c.ociStore, config.Runtimes) + factory.SetOciStore(c.ociStore) + factory.ProvisionRuntime(config.Runtimes) if sb, err := factory.Create(); err != nil { return err } else {