diff --git a/internal/builder/vm/lcow/kernel_args.go b/internal/builder/vm/lcow/kernel_args.go index d5822b73b8..d828226aba 100644 --- a/internal/builder/vm/lcow/kernel_args.go +++ b/internal/builder/vm/lcow/kernel_args.go @@ -27,7 +27,6 @@ func buildKernelArgs( kernelDirect bool, hasConsole bool, rootFsFile string, - liveMigrationSupportEnabled bool, ) (string, error) { log.G(ctx).WithField("rootFsFile", rootFsFile).Debug("buildKernelArgs: starting kernel arguments construction") @@ -83,7 +82,7 @@ func buildKernelArgs( // 8. Init arguments (passed after "--" separator) initArgs := buildInitArgs(ctx, opts, annotations, - writableOverlayDirs, disableTimeSyncService, processDumpLocation, rootFsFile, hasConsole, liveMigrationSupportEnabled) + writableOverlayDirs, disableTimeSyncService, processDumpLocation, rootFsFile, hasConsole) args = append(args, "--", initArgs) result := strings.Join(args, " ") @@ -153,7 +152,6 @@ func buildInitArgs( processDumpLocation string, rootFsFile string, hasConsole bool, - liveMigrationSupportEnabled bool, ) string { log.G(ctx).WithFields(logrus.Fields{ "rootFsFile": rootFsFile, @@ -163,7 +161,7 @@ func buildInitArgs( entropyArgs := fmt.Sprintf("-e %d", vmutils.LinuxEntropyVsockPort) // Build GCS execution command - gcsCmd := buildGCSCommand(opts, annotations, disableTimeSyncService, processDumpLocation, liveMigrationSupportEnabled) + gcsCmd := buildGCSCommand(ctx, opts, annotations, disableTimeSyncService, processDumpLocation) // Construct init arguments var initArgsList []string @@ -194,12 +192,26 @@ func buildInitArgs( // buildGCSCommand constructs the GCS (Guest Compute Service) command line. func buildGCSCommand( + ctx context.Context, opts *runhcsoptions.Options, annotations map[string]string, disableTimeSyncService bool, processDumpLocation string, - liveMigrationSupportEnabled bool, ) string { + // Start with vsockexec wrapper + var cmdParts []string + cmdParts = append(cmdParts, "/bin/vsockexec") + + // When live migration is enabled, run vsockexec in reconnect mode (-r) so + // guest logging tolerates the host log listener being absent at boot and + // reconnects to the destination host's listener after a migration. + if oci.ParseAnnotationsBool(ctx, annotations, shimannotations.LiveMigrationSupportEnabled, false) { + cmdParts = append(cmdParts, "-r") + } + + // Add logging vsock port + cmdParts = append(cmdParts, fmt.Sprintf("-e %d", vmutils.LinuxLogVsockPort)) + // Determine log level logLevel := "info" if opts != nil && opts.LogLevel != "" { @@ -232,19 +244,8 @@ func buildGCSCommand( gcsParts = append(gcsParts, s) } - gcsCmd := strings.Join(gcsParts, " ") - - // Live-migratable pods skip the /bin/vsockexec wrapper. The wrapper exists - // solely to forward GCS stderr to the host-side log listener, but that listener - // is host-local state that live migration does not transfer, so the host - // does not run it for these pods. - // Without a listener, vsockexec's outbound connect would block and stall guest init, - // so we emit /bin/gcs directly instead. - if liveMigrationSupportEnabled { - return gcsCmd - } + // Combine vsockexec and GCS command + cmdParts = append(cmdParts, strings.Join(gcsParts, " ")) - // vsockexec `-e ` wires gcs's stderr to LinuxLogVsockPort, which - // the host listener reads and republishes. - return fmt.Sprintf("/bin/vsockexec -e %d %s", vmutils.LinuxLogVsockPort, gcsCmd) + return strings.Join(cmdParts, " ") } diff --git a/internal/builder/vm/lcow/sandbox_options.go b/internal/builder/vm/lcow/sandbox_options.go index 37c978a170..492d3678b5 100644 --- a/internal/builder/vm/lcow/sandbox_options.go +++ b/internal/builder/vm/lcow/sandbox_options.go @@ -25,10 +25,6 @@ type SandboxOptions struct { // ConfidentialConfig carries confidential computing fields that are not // part of the HCS document but are needed for confidential VM setup. ConfidentialConfig *ConfidentialConfig - - // LiveMigrationSupportEnabled indicates that the live migration feature set is - // enabled for the sandbox, constraining it to migration-compatible features. - LiveMigrationSupportEnabled bool } // ConfidentialConfig carries confidential computing configuration that is not diff --git a/internal/builder/vm/lcow/specs.go b/internal/builder/vm/lcow/specs.go index 18074e3bc4..2229e0da63 100644 --- a/internal/builder/vm/lcow/specs.go +++ b/internal/builder/vm/lcow/specs.go @@ -222,7 +222,6 @@ func BuildSandboxConfig( bootOptions.LinuxKernelDirect != nil, // isKernelDirectBoot comPorts != nil, // hasConsole filepath.Base(rootFsFullPath), - sandboxOptions.LiveMigrationSupportEnabled, ) if err != nil { return nil, nil, fmt.Errorf("failed to build kernel args: %w", err) @@ -331,11 +330,10 @@ func parseSandboxOptions(ctx context.Context, platform string, annotations map[s log.G(ctx).WithField("platform", platform).Debug("parseSandboxOptions: starting sandbox options parsing") sandboxOptions := &SandboxOptions{ // Extract architecture from platform string (e.g., "linux/amd64" -> "amd64") - Architecture: platform[strings.IndexByte(platform, '/')+1:], - FullyPhysicallyBacked: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.FullyPhysicallyBacked, false), - PolicyBasedRouting: oci.ParseAnnotationsBool(ctx, annotations, iannotations.NetworkingPolicyBasedRouting, false), - NoWritableFileShares: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DisableWritableFileShares, false), - LiveMigrationSupportEnabled: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.LiveMigrationSupportEnabled, false), + Architecture: platform[strings.IndexByte(platform, '/')+1:], + FullyPhysicallyBacked: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.FullyPhysicallyBacked, false), + PolicyBasedRouting: oci.ParseAnnotationsBool(ctx, annotations, iannotations.NetworkingPolicyBasedRouting, false), + NoWritableFileShares: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DisableWritableFileShares, false), } // Determine if this is a confidential VM early, as it affects boot options parsing diff --git a/internal/builder/vm/lcow/specs_test.go b/internal/builder/vm/lcow/specs_test.go index ac7d0b5842..6b04df1ff2 100644 --- a/internal/builder/vm/lcow/specs_test.go +++ b/internal/builder/vm/lcow/specs_test.go @@ -2129,34 +2129,35 @@ func TestBuildSandboxConfig_CPUClamping(t *testing.T) { } // TestBuildSandboxConfig_LiveMigration validates the wiring for the -// io.microsoft.migration.support-enabled sandbox annotation. The annotation is parsed -// into SandboxOptions.LiveMigrationSupportEnabled and threaded down into the kernel -// command line: live-migratable sandboxes must skip the /bin/vsockexec wrapper -// (which would otherwise stall init waiting for a host log listener that the -// LM-enabled host does not run), while non-LM sandboxes must continue to use -// vsockexec so that GCS stderr is forwarded over LinuxLogVsockPort. +// io.microsoft.migration.support-enabled sandbox annotation. The annotation is +// parsed directly when building the GCS command: live-migratable sandboxes run +// the /bin/vsockexec wrapper in reconnect mode (-r) so guest logging tolerates a +// missing host log listener and reconnects after a migration, while non-LM +// sandboxes use the plain wrapper to forward GCS stderr over LinuxLogVsockPort. func TestBuildSandboxConfig_LiveMigration(t *testing.T) { ctx := context.Background() validBootFilesPath := newBootFilesPath(t) defaultOpts := defaultSandboxOpts(validBootFilesPath) - // Pre-format the vsockexec prefix once so the assertions are obviously - // driven by the same constant the production code uses. + // Pre-format the vsockexec prefixes once so the assertions are obviously + // driven by the same constants the production code uses. vsockexecPrefix := fmt.Sprintf("/bin/vsockexec -e %d", vmutils.LinuxLogVsockPort) + // When live migration is enabled the wrapper runs in reconnect mode (-r). + vsockexecReconnectPrefix := fmt.Sprintf("/bin/vsockexec -r -e %d", vmutils.LinuxLogVsockPort) tests := []specTestCase{ { name: "live migration disabled by default", validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=false by default, got true") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, vsockexecPrefix) { t.Errorf("expected vsockexec wrapper %q in kernel args (LM disabled), got %q", vsockexecPrefix, kernelArgs) } + if strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected no vsockexec reconnect mode when LM disabled, got %q", kernelArgs) + } if !strings.Contains(kernelArgs, "/bin/gcs") { t.Errorf("expected /bin/gcs in kernel args, got %q", kernelArgs) } @@ -2171,17 +2172,17 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=false when annotation=\"false\", got true") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, vsockexecPrefix) { t.Errorf("expected vsockexec wrapper %q in kernel args, got %q", vsockexecPrefix, kernelArgs) } + if strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected no vsockexec reconnect mode when LM disabled, got %q", kernelArgs) + } }, }, { - name: "live migration enabled drops vsockexec wrapper", + name: "live migration enabled uses vsockexec reconnect mode", spec: &vm.Spec{ Annotations: map[string]string{ shimannotations.LiveMigrationSupportEnabled: "true", @@ -2189,21 +2190,14 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if !sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=true when annotation=\"true\", got false") - } kernelArgs := getKernelArgs(doc) - // The vsockexec wrapper must not appear at all when LM is on: - // neither the prefix nor the binary path on its own. - if strings.Contains(kernelArgs, "vsockexec") { - t.Errorf("expected no vsockexec in kernel args when LM enabled, got %q", kernelArgs) + // When LM is on the wrapper runs in reconnect mode (-r) so the + // guest tolerates a missing or dropped host log listener. + if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected vsockexec reconnect wrapper %q in kernel args when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs) } - if strings.Contains(kernelArgs, fmt.Sprintf("-e %d", vmutils.LinuxLogVsockPort)) { - t.Errorf("expected no log vsock port (%d) wiring when LM enabled, got %q", vmutils.LinuxLogVsockPort, kernelArgs) - } - // /bin/gcs must still be invoked - just without the wrapper. if !strings.Contains(kernelArgs, "/bin/gcs") { - t.Errorf("expected /bin/gcs in kernel args even when LM enabled, got %q", kernelArgs) + t.Errorf("expected /bin/gcs in kernel args when LM enabled, got %q", kernelArgs) } }, }, @@ -2221,22 +2215,19 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if !sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=true, got false") - } kernelArgs := getKernelArgs(doc) - // Other GCS flags must still be threaded through the command - // even when the vsockexec wrapper is removed. + // Other GCS flags must still be threaded through the reconnect + // wrapper when LM is enabled. if !strings.Contains(kernelArgs, "-loglevel debug") { t.Errorf("expected -loglevel debug in kernel args when LM enabled, got %q", kernelArgs) } - if strings.Contains(kernelArgs, "vsockexec") { - t.Errorf("expected no vsockexec when LM enabled, got %q", kernelArgs) + if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected vsockexec reconnect wrapper %q when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs) } }, }, { - name: "live migration with disable time sync still drops vsockexec", + name: "live migration with disable time sync uses vsockexec reconnect mode", spec: &vm.Spec{ Annotations: map[string]string{ shimannotations.LiveMigrationSupportEnabled: "true", @@ -2245,15 +2236,12 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if !sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=true, got false") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, "-disable-time-sync") { t.Errorf("expected -disable-time-sync flag in kernel args, got %q", kernelArgs) } - if strings.Contains(kernelArgs, "vsockexec") { - t.Errorf("expected no vsockexec when LM enabled, got %q", kernelArgs) + if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected vsockexec reconnect wrapper %q when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs) } }, }, @@ -2269,13 +2257,13 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=false on invalid annotation value, got true") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, vsockexecPrefix) { t.Errorf("expected vsockexec wrapper %q in kernel args, got %q", vsockexecPrefix, kernelArgs) } + if strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected no vsockexec reconnect mode on invalid LM annotation, got %q", kernelArgs) + } }, }, } diff --git a/internal/controller/vm/save_lcow.go b/internal/controller/vm/save_lcow.go index 29f330c74a..df4f0ee1e5 100644 --- a/internal/controller/vm/save_lcow.go +++ b/internal/controller/vm/save_lcow.go @@ -18,6 +18,7 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/go-winio" + "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) @@ -222,6 +223,27 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error { if err := c.guest.ResumeConnection(ctx); err != nil { return fmt.Errorf("resume guest connection: %w", err) } + + // The blackout also dropped the source's GCS log connection, which tore + // down its listener and closed logOutputDone. Install a fresh signal and + // re-arm the listener so the resumed guest's reconnect-mode vsockexec can + // reconnect and host-side logs resume. The accept runs in the background + // (WithoutCancel so it outlives this call; AcceptConnection still returns + // on VM exit) so a slow guest re-dial cannot stall resume. + c.logOutputDone = make(chan struct{}) + g, gctx := errgroup.WithContext(ctx) + defer func() { + _ = g.Wait() + }() + + if err := c.setupLoggingListener(gctx, g); err != nil { + return fmt.Errorf("re-arm logging listener on resume: %w", err) + } + + // Collect any errors from establishing the log connection. + if err := g.Wait(); err != nil { + return err + } default: // Destination: reuse the connection already armed at start. if err := c.guest.CreateConnection(ctx, false); err != nil { @@ -252,18 +274,6 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error { c.vmState = StateRunning - if c.sandboxOptions != nil { - c.sandboxOptions.LiveMigrationSupportEnabled = true - } - - // Destination never ran setupLoggingListener; close so [Controller.Wait] - // does not block. Already closed on source — receive falls through. - select { - case <-c.logOutputDone: - default: - close(c.logOutputDone) - } - log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("resumed VM from migration") return nil } diff --git a/internal/controller/vm/vm_lcow.go b/internal/controller/vm/vm_lcow.go index 3c548df04c..7c5aad253f 100644 --- a/internal/controller/vm/vm_lcow.go +++ b/internal/controller/vm/vm_lcow.go @@ -12,7 +12,6 @@ import ( "github.com/Microsoft/hcsshim/internal/controller/device/plan9" "github.com/Microsoft/hcsshim/internal/controller/network" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" - "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/vm/vmmanager" "github.com/Microsoft/hcsshim/internal/vm/vmutils" @@ -174,30 +173,10 @@ func (c *Controller) setupEntropyListener(ctx context.Context, group *errgroup.G // running inside the Linux VM. The logs are parsed and // forwarded to the host's logging system for monitoring and debugging. func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.Group) error { - // Live-migratable sandboxes intentionally run without a host-side GCS log - // listener. - // - // The log listener is host-local state: GCS inside the guest connects out to - // a host-side hvsocket on LinuxLogVsockPort and streams its stderr to it. That - // connection, and the goroutine reading from it, are bound to the *source* - // host and are not part of the guest state that live migration transfers. - // After the VM is migrated to a destination host there is no equivalent - // listener to reconnect to, so a guest that depended on the log socket would - // block on its outbound connect and stall the boot path. To keep the guest - // migratable we skip the listener here and drop the matching /bin/vsockexec - // wrapper from the kernel command line, so GCS never attempts the connection. - // - // Re-enabling host-side log collection for live-migratable pods requires a - // migration-aware log transport: GCS must tolerate the listener going away - // and reconnect to a freshly established listener on the destination host once - // migration completes, and the host must (re)create the listener and re-attach - // the log-parsing goroutine on the destination. Until that work lands we forgo - // host-side GCS logs for these pods. - if c.sandboxOptions != nil && c.sandboxOptions.LiveMigrationSupportEnabled { - log.G(ctx).Info("skipping GCS log listener: pod is live-migratable") - close(c.logOutputDone) - return nil - } + // Capture the current signal channel: if the listener is re-armed later + // (e.g. source rollback), this goroutine must close the channel it was + // started with, never a freshly installed one. + done := c.logOutputDone // The GCS will connect to this port to stream log output. logConn, err := winio.ListenHvsock(&winio.HvsockAddr{ @@ -205,7 +184,7 @@ func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.G ServiceID: winio.VsockServiceID(vmutils.LinuxLogVsockPort), }) if err != nil { - close(c.logOutputDone) + close(done) return fmt.Errorf("failed to listen on hvSocket for logs: %w", err) } @@ -215,7 +194,7 @@ func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.G // Accept the connection from the GCS. conn, err := vmmanager.AcceptConnection(ctx, c.uvm, logConn, true) if err != nil { - close(c.logOutputDone) + close(done) return fmt.Errorf("failed to accept connection on hvSocket for logs: %w", err) } @@ -228,7 +207,7 @@ func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.G // Signal that log output processing has completed. // This allows Wait() to ensure all logs are processed before returning. - close(c.logOutputDone) + close(done) }() return nil diff --git a/internal/controller/vm/vm_migration.go b/internal/controller/vm/vm_migration.go index 267669475a..6d3b8e86c4 100644 --- a/internal/controller/vm/vm_migration.go +++ b/internal/controller/vm/vm_migration.go @@ -6,14 +6,17 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/Microsoft/go-winio" + "golang.org/x/sync/errgroup" "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/internal/vm/vmutils" ) // compatibilityInfoProperty is the HCS property name used to retrieve the @@ -32,6 +35,20 @@ func (c *Controller) InitializeLiveMigrationOnSource(ctx context.Context, option return fmt.Errorf("cannot initialize live migration on source: VM is in state %s", c.vmState) } + // Live migration requires the guest log relay to run in reconnect mode so it can + // re-dial the destination host's log listener after the move. + var kernelCmdLine string + if chipset := c.hcsDocument.VirtualMachine.Chipset; chipset != nil { + if chipset.LinuxKernelDirect != nil { + kernelCmdLine = chipset.LinuxKernelDirect.KernelCmdLine + } else if chipset.Uefi != nil && chipset.Uefi.BootThis != nil { + kernelCmdLine = chipset.Uefi.BootThis.OptionalData + } + } + if !strings.Contains(kernelCmdLine, fmt.Sprintf("/bin/vsockexec -r -e %d", vmutils.LinuxLogVsockPort)) { + return fmt.Errorf("cannot initialize live migration on source: VM was not created with live-migration support enabled") + } + // Hand the initialize request to the HCS for the UVM. if err := c.uvm.InitializeLiveMigrationOnSource(ctx, options); err != nil { return fmt.Errorf("failed to initialize live migration on source: %w", err) @@ -115,6 +132,19 @@ func (c *Controller) StartWithMigrationOptions(ctx context.Context, config *hcs. return fmt.Errorf("cannot start with migration options: VM is in state %s", c.vmState) } + g, gctx := errgroup.WithContext(ctx) + defer func() { + _ = g.Wait() + }() + + // The source's log listener is host-local and does not transfer with the + // migration. Arm a fresh listener on the destination before start so the + // resumed guest's vsockexec (reconnect mode) can re-dial LinuxLogVsockPort + // and resume streaming GCS logs without racing the guest's dial. + if err := c.setupLoggingListener(gctx, g); err != nil { + return fmt.Errorf("failed to set up logging listener: %w", err) + } + // Arm the host-side GCS listener before start so the guest's dial cannot race it. if err := c.guest.PrepareConnection(winio.VsockServiceID(prot.LinuxGcsVsockPort)); err != nil { return fmt.Errorf("prepare destination gcs connection: %w", err) @@ -127,6 +157,12 @@ func (c *Controller) StartWithMigrationOptions(ctx context.Context, config *hcs. // Watch for VM exit in the background. go c.waitForVMExit(ctx) + + // Collect any errors from establishing the log connection. + if err := g.Wait(); err != nil { + return err + } + c.vmState = StateDestinationMigrationStarted log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("started destination VM with migration options") diff --git a/vsockexec/vsockexec.c b/vsockexec/vsockexec.c index fa71b2e6ea..0677a97e20 100644 --- a/vsockexec/vsockexec.c +++ b/vsockexec/vsockexec.c @@ -1,13 +1,21 @@ // vsockexec opens vsock connections for the specified stdio descriptors and // then execs the specified process. +// +// With -r ("reconnect mode") it does NOT hand the connection to the process. +// Instead it keeps running as a middleman (a "relay") between the process and +// the host, so that a host connection which is missing at startup or which +// drops later never reaches the process. See run_relay below for the details. #include "vsock.h" #include #include +#include #include #include #include #include +#include +#include #include #ifdef USE_TCP @@ -33,16 +41,171 @@ static int opentcp(unsigned short port) { return s; } +// sleep_ms pauses for the given number of milliseconds. We use it to wait a +// short moment between connection attempts instead of retrying back-to-back and +// burning CPU. +static void sleep_ms(long ms) { + struct timespec ts = {ms / 1000, (ms % 1000) * 1000000L}; + nanosleep(&ts, NULL); +} + +// dial_retry opens a connection to the host on the given port and never gives +// up. If the host side isn't listening yet (it may come up a little later) or +// has gone away (for example the VM was moved to a different host), it just +// keeps trying until it succeeds. This is what lets the guest wait for the host +// instead of failing, which is important because failing here would take the +// whole VM down. +static int dial_retry(unsigned int port) { + for (;;) { + // Try to connect once. Normal builds use vsock to reach the host; the + // tcp branch only exists for local testing. + int s = tcpmode ? opentcp(port) : openvsock(VMADDR_CID_HOST, port); + if (s >= 0) { + return s; // success: we now have a live connection to the host + } + // Couldn't connect yet. Wait a moment and then try again. + sleep_ms(100); + } +} + +// run_relay implements "reconnect mode" (-r). +// +// Normally vsockexec connects a socket to the host and hands that socket +// straight to the program as its stdout/stderr. The problem: if that host +// connection breaks, the program's writes fail and the program dies. Since the +// program is a child of PID 1 inside the VM, a dying program makes PID 1 exit, +// which makes the Linux kernel panic and the VM crash. +// +// run_relay avoids that by sitting in the MIDDLE and never letting the program +// touch the host connection directly: +// +// program --writes--> [ pipe ] --run_relay reads--> [ host connection ] +// +// The program only ever writes into a pipe, which never breaks. run_relay reads +// from that pipe and forwards the data to the host. If the host connection is +// missing or drops, only run_relay notices; it quietly reconnects and keeps +// going, so the program never sees a problem and stays alive. +static int run_relay(unsigned int ports[3], char** child_argv) { + // In reconnect mode we forward the program's OUTPUT (stdout and/or stderr) + // to one host port. Use whichever output port was requested. + unsigned int port = ports[2] != 0 ? ports[2] : ports[1]; + if (port == 0) { + // Nothing to forward: reconnect mode is only meaningful with -o or -e. + fprintf(stderr, "vsockexec: -r requires -o or -e\n"); + return 1; + } + + // Writing to a connection whose other end has gone away normally kills this + // process with a SIGPIPE signal. Tell the system to ignore that signal, so + // a dropped connection instead shows up as an ordinary write error that we + // can catch below and recover from by reconnecting. + signal(SIGPIPE, SIG_IGN); + + // Create a pipe: a simple one-way channel. Anything written to p[1] (the + // write end) can be read back from p[0] (the read end). The program will + // write into p[1]; we (the relay) will read from p[0]. + int p[2]; + if (pipe(p) < 0) { + perror("pipe"); + return 1; + } + + // Split into two processes. The child will turn into the program; the + // parent stays behind and becomes the relay/middleman. + pid_t pid = fork(); + if (pid < 0) { + perror("fork"); + return 1; + } + if (pid == 0) { + // ---------------- Child: becomes the program (e.g. GCS) ------------- + signal(SIGPIPE, SIG_DFL); // restore normal signal behavior for the program + close(p[0]); // the program only writes, so close the read end + // Redirect the program's stdout/stderr into the pipe instead of a real + // socket. After this, everything the program prints goes into the pipe. + if (ports[1] != 0) { + dup2(p[1], 1); // stdout -> pipe + } + if (ports[2] != 0) { + dup2(p[1], 2); // stderr -> pipe + } + close(p[1]); // the fds above are now our copies; close the original + // Replace this child process with the requested program and run it. + // CodeQL [SM01925] designed to forward stdio over VSOCK and then exec the command-line arguments (always ./cmd/gcs) + execvp(child_argv[0], child_argv); + // We only reach here if the program failed to start. + fprintf(stderr, "execvp: %s: %s\n", child_argv[0], strerror(errno)); + _exit(127); + } + + // -------------------- Parent: the relay/middleman ---------------------- + close(p[1]); // the relay only reads, so close the write end + + // Open the connection to the host, waiting for the host if it isn't ready. + int sock = dial_retry(port); + + // Main loop: keep copying whatever the program writes into the pipe out to + // the host connection, for as long as the program is running. + char buf[65536]; + for (;;) { + // Read the next chunk of the program's output from the pipe. + ssize_t n = read(p[0], buf, sizeof(buf)); + if (n < 0) { + if (errno == EINTR) { + continue; // a signal interrupted the read; simply try again + } + break; // unexpected error reading the pipe; stop + } + if (n == 0) { + break; // the pipe reached end-of-file: the program has exited + } + // Send those n bytes to the host. Keep sending until every byte is out, + // reconnecting if the connection breaks in the middle of sending. + for (ssize_t off = 0; off < n;) { + ssize_t w = write(sock, buf + off, (size_t)(n - off)); + if (w < 0) { + if (errno == EINTR) { + continue; // interrupted by a signal; retry the write + } + // The connection dropped (for example the host went away). Drop + // the dead socket, get a fresh connection, and resend the bytes + // that had not been sent yet (off marks how far we got). + close(sock); + sock = dial_retry(port); + continue; + } + off += w; // w bytes were sent; move past them and continue + } + } + close(sock); + + // The program has finished. Wait for it and then exit with the SAME status + // it did, so that whoever launched us (PID 1) sees the program's real exit + // code and behaves exactly as if it had run the program directly. + int status = 0; + while (waitpid(pid, &status, 0) < 0 && errno == EINTR) { + // a signal interrupted the wait; keep waiting + } + if (WIFEXITED(status)) { + return WEXITSTATUS(status); // program exited normally: pass on its code + } + if (WIFSIGNALED(status)) { + return 128 + WTERMSIG(status); // program was killed by a signal + } + return 1; +} + _Noreturn static void usage(const char* argv0) { - fprintf(stderr, "%s [-i port] [-o port] [-e port] -- program [args...]\n", argv0); + fprintf(stderr, "%s [-r] [-i port] [-o port] [-e port] -- program [args...]\n", argv0); exit(1); } int main(int argc, char** argv) { unsigned int ports[3] = {0}; int sockets[3] = {-1, -1, -1}; + int reconnect = 0; int c; - while ((c = getopt(argc, argv, "+i:o:e:")) != -1) { + while ((c = getopt(argc, argv, "+i:o:e:r")) != -1) { switch (c) { case 'i': ports[0] = strtoul(optarg, NULL, 10); @@ -56,6 +219,11 @@ int main(int argc, char** argv) { ports[2] = strtoul(optarg, NULL, 10); break; + case 'r': + // -r turns on reconnect mode (resilient forwarding via run_relay). + reconnect = 1; + break; + default: usage(argv[0]); } @@ -66,6 +234,13 @@ int main(int argc, char** argv) { usage(argv[0]); } + if (reconnect) { + // Resilient path: wait for the host if it isn't listening yet and + // reconnect if the connection later drops. The plain connect-and-exec + // path below (used when -r is absent) does neither. + return run_relay(ports, argv + optind); + } + for (int i = 0; i < 3; i++) { if (ports[i] != 0) { int j;