batches: Remote Mount #5 - src-cli (#816)

Author: Piszmog

CHANGELOG.md

@@ -13,6 +13,8 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Added
 
+- Mounting files now works when running batch changes server side. [sourcegraph/src-cli#816](https://github.com/sourcegraph/src-cli/pull/816)
+
 ### Changed
 
 ### Fixed
@@ -857,4 +859,4 @@ Re-release of 3.29.3 for Sourcegraph 3.30.
 ### Changed
 
 - The terminal UI has been replaced by the logger-based UI that was previously only visible in verbose-mode (`-v`). [#228](https://github.com/sourcegraph/src-cli/pull/228)
-- Deprecated the `-endpoint` flag. Instead, use the `SRC_ENDPOINT` environment variable. [#235](https://github.com/sourcegraph/src-cli/pull/235)
+- Deprecated the `-endpoint` flag. Instead, use the `SRC_ENDPOINT` environment variable. [#235](https://github.com/sourcegraph/src-cli/pull/235)

cmd/src/batch_common.go

@@ -304,7 +304,7 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp
 
 	// Parse flags and build up our service and executor options.
 	ui.ParsingBatchSpec()
-	batchSpec, rawSpec, err := parseBatchSpec(ctx, opts.file, svc, false)
+	batchSpec, batchSpecDir, rawSpec, err := parseBatchSpec(ctx, opts.file, svc)
 	if err != nil {
 		var multiErr errors.MultiError
 		if errors.As(err, &multiErr) {
@@ -378,15 +378,14 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp
 				Creator:             workspaceCreator,
 				EnsureImage:         imageCache.Ensure,
 				Parallelism:         parallelism,
+				WorkingDirectory:    batchSpecDir,
 				Timeout:             opts.flags.timeout,
 				TempDir:             opts.flags.tempDir,
 				GlobalEnv:           os.Environ(),
-				IsRemote:            false,
 			},
 			Logger:    logManager,
 			Cache:     executor.NewDiskCache(opts.flags.cacheDir),
 			GlobalEnv: os.Environ(),
-			IsRemote:  false,
 		},
 	)
 
@@ -508,13 +507,10 @@ func setReadDeadlineOnCancel(ctx context.Context, f *os.File) {
 
 // parseBatchSpec parses and validates the given batch spec. If the spec has
 // validation errors, they are returned.
-//
-// isRemote argument is a temporary argument used to determine if the batch spec is being parsed for remote
-// (server-side) processing. Remote processing does not support mounts yet.
-func parseBatchSpec(ctx context.Context, file string, svc *service.Service, isRemote bool) (*batcheslib.BatchSpec, string, error) {
+func parseBatchSpec(ctx context.Context, file string, svc *service.Service) (*batcheslib.BatchSpec, string, string, error) {
 	f, err := batchOpenFileFlag(file)
 	if err != nil {
-		return nil, "", err
+		return nil, "", "", err
 	}
 	defer f.Close()
 
@@ -526,26 +522,35 @@ func parseBatchSpec(ctx context.Context, file string, svc *service.Service, isRe
 
 	data, err := io.ReadAll(f)
 	if err != nil {
-		return nil, "", errors.Wrap(err, "reading batch spec")
+		return nil, "", "", errors.Wrap(err, "reading batch spec")
 	}
 
+	dir, err := getBatchSpecDirectory(file)
+	if err != nil {
+		return nil, "", "", errors.Wrap(err, "batch spec path")
+	}
+
+	spec, err := svc.ParseBatchSpec(dir, data)
+	return spec, dir, string(data), err
+}
+
+func getBatchSpecDirectory(file string) (string, error) {
 	var workingDirectory string
+	var err error
 	// if the batch spec is being provided via standard input, set the working directory to the current directory
 	if file == "" || file == "-" {
 		workingDirectory, err = os.Getwd()
 		if err != nil {
-			return nil, "", errors.Wrap(err, "batch spec path")
+			return "", errors.Wrap(err, "batch spec path")
 		}
 	} else {
 		p, err := filepath.Abs(file)
 		if err != nil {
-			return nil, "", errors.Wrap(err, "batch spec path")
+			return "", errors.Wrap(err, "batch spec path")
 		}
 		workingDirectory = filepath.Dir(p)
 	}
-
-	spec, err := svc.ParseBatchSpec(workingDirectory, data, isRemote)
-	return spec, string(data), err
+	return workingDirectory, nil
 }
 
 func checkExecutable(cmd string, args ...string) error {

cmd/src/batch_exec.go

@@ -10,16 +10,17 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/sourcegraph/src-cli/internal/batches/docker"
+	"github.com/sourcegraph/src-cli/internal/batches/log"
+	"github.com/sourcegraph/src-cli/internal/batches/repozip"
+	"github.com/sourcegraph/src-cli/internal/batches/workspace"
+
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 
-	"github.com/sourcegraph/src-cli/internal/batches/docker"
 	"github.com/sourcegraph/src-cli/internal/batches/executor"
 	"github.com/sourcegraph/src-cli/internal/batches/graphql"
-	"github.com/sourcegraph/src-cli/internal/batches/log"
-	"github.com/sourcegraph/src-cli/internal/batches/repozip"
 	"github.com/sourcegraph/src-cli/internal/batches/service"
 	"github.com/sourcegraph/src-cli/internal/batches/ui"
-	"github.com/sourcegraph/src-cli/internal/batches/workspace"
 	"github.com/sourcegraph/src-cli/internal/cmderrors"
 
 	batcheslib "github.com/sourcegraph/sourcegraph/lib/batches"
@@ -30,10 +31,11 @@ const (
 )
 
 type executorModeFlags struct {
-	timeout time.Duration
-	file    string
-	tempDir string
-	repoDir string
+	timeout           time.Duration
+	file              string
+	tempDir           string
+	repoDir           string
+	workspaceFilesDir string
 }
 
 func newExecutorModeFlags(flagSet *flag.FlagSet) (f *executorModeFlags) {
@@ -42,6 +44,7 @@ func newExecutorModeFlags(flagSet *flag.FlagSet) (f *executorModeFlags) {
 	flagSet.StringVar(&f.file, "f", "", "The workspace execution input file to read.")
 	flagSet.StringVar(&f.tempDir, "tmp", "", "Directory for storing temporary data.")
 	flagSet.StringVar(&f.repoDir, "repo", "", "Path of the checked out repo on disk.")
+	flagSet.StringVar(&f.workspaceFilesDir, "workspaceFiles", "", "Path of workspace files on disk.")
 
 	return f
 }
@@ -69,7 +72,7 @@ github.com/sourcegraph/sourcegraph/lib/batches.
 
 Usage:
 
-    src batch exec -f FILE -repo DIR [command options]
+    src batch exec -f FILE -repo DIR -workspaceFiles DIR [command options]
 
 Examples:
 
@@ -140,6 +143,12 @@ func executeBatchSpecInWorkspaces(ctx context.Context, flags *executorModeFlags)
 		}
 	}
 
+	// Grab the absolute path to the workspace files contents.
+	workspaceFilesDir, err := filepath.Abs(flags.workspaceFilesDir)
+	if err != nil {
+		return errors.Wrap(err, "getting absolute path for workspace files dir")
+	}
+
 	// Test if git is available.
 	if err := checkExecutable("git", "version"); err != nil {
 		return err
@@ -177,8 +186,7 @@ func executeBatchSpecInWorkspaces(ctx context.Context, flags *executorModeFlags)
 	ui.PreparingContainerImagesSuccess()
 
 	// Empty for now until we support secrets or env var settings in SSBC.
-	globalEnv := []string{}
-	isRemote := true
+	var globalEnv []string
 
 	// Set up the execution UI.
 	taskExecUI := ui.ExecutingTasks(false, 1)
@@ -191,19 +199,18 @@ func executeBatchSpecInWorkspaces(ctx context.Context, flags *executorModeFlags)
 		EnsureImage: imageCache.Ensure,
 		Task:        task,
 		// TODO: Should be slightly less than the executor timeout. Can we somehow read that?
-		Timeout:   flags.timeout,
-		TempDir:   tempDir,
-		GlobalEnv: globalEnv,
-		// Temporarily prevent the ability to sending a batch spec with a mount for server-side processing.
-		IsRemote:    isRemote,
-		RepoArchive: &repozip.NoopArchive{},
-		UI:          taskExecUI.StepsExecutionUI(task),
+		Timeout:          flags.timeout,
+		TempDir:          tempDir,
+		WorkingDirectory: workspaceFilesDir,
+		GlobalEnv:        globalEnv,
+		RepoArchive:      &repozip.NoopArchive{},
+		UI:               taskExecUI.StepsExecutionUI(task),
 	}
 	results, err := executor.RunSteps(ctx, opts)
 
 	// Write all step cache results for all results.
 	for _, stepRes := range results {
-		cacheKey := task.CacheKey(globalEnv, isRemote, stepRes.StepIndex)
+		cacheKey := task.CacheKey(globalEnv, workspaceFilesDir, stepRes.StepIndex)
 		k, err := cacheKey.Key()
 		if err != nil {
 			return errors.Wrap(err, "calculating step cache key")

cmd/src/batch_remote.go

@@ -63,7 +63,7 @@ Examples:
 		// may as well validate it at the same time so we don't even have to go to
 		// the backend if it's invalid.
 		ui.ParsingBatchSpec()
-		spec, raw, err := parseBatchSpec(ctx, file, svc, true)
+		spec, batchSpecDir, raw, err := parseBatchSpec(ctx, file, svc)
 		if err != nil {
 			ui.ParsingBatchSpecFailure(err)
 			return err
@@ -100,6 +100,21 @@ Examples:
 		}
 		ui.SendingBatchSpecSuccess()
 
+		hasWorkspaceFiles := false
+		for _, step := range spec.Steps {
+			if len(step.Mount) > 0 {
+				hasWorkspaceFiles = true
+				break
+			}
+		}
+		if hasWorkspaceFiles {
+			ui.UploadingWorkspaceFiles()
+			if err = svc.UploadBatchSpecWorkspaceFiles(ctx, batchSpecDir, batchSpecID, spec.Steps); err != nil {
+				return err
+			}
+			ui.UploadingWorkspaceFilesSuccess()
+		}
+
 		// Wait for the workspaces to be resolved.
 		ui.ResolvingWorkspaces()
 		ticker := time.NewTicker(1 * time.Second)

cmd/src/batch_repositories.go

@@ -74,7 +74,7 @@ Examples:
 		}
 
 		out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
-		spec, _, err := parseBatchSpec(ctx, file, svc, false)
+		spec, _, _, err := parseBatchSpec(ctx, file, svc)
 		if err != nil {
 			ui := &ui.TUI{Out: out}
 			ui.ParsingBatchSpecFailure(err)

cmd/src/batch_validate.go

@@ -73,7 +73,7 @@ Examples:
 			return err
 		}
 
-		if _, _, err := parseBatchSpec(ctx, file, svc, false); err != nil {
+		if _, _, _, err := parseBatchSpec(ctx, file, svc); err != nil {
 			ui.ParsingBatchSpecFailure(err)
 			return err
 		}

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/sourcegraph/go-diff v0.6.1
 	github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
 	github.com/sourcegraph/scip v0.2.1
-	github.com/sourcegraph/sourcegraph/lib v0.0.0-20220825181731-397a768a5290
+	github.com/sourcegraph/sourcegraph/lib v0.0.0-20221004162410-237d5855fa13
 	github.com/stretchr/testify v1.8.0
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
@@ -105,7 +105,7 @@ require (
 	go.uber.org/zap v1.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
+	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
 	golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.12 // indirect

go.sum

@@ -373,8 +373,8 @@ github.com/sourcegraph/log v0.0.0-20220901143117-fc0516a694c9 h1:JjFyvx9hCD5+Jpu
 github.com/sourcegraph/log v0.0.0-20220901143117-fc0516a694c9/go.mod h1:UxiwB6C3xk3xOySJpW1R0MDUyfGuJRFS5Z8C+SA5p2I=
 github.com/sourcegraph/scip v0.2.1 h1:t4cTR8TU81Psfmcj3ZYer67JFzsR/Sc7w5rsuckRo1Y=
 github.com/sourcegraph/scip v0.2.1/go.mod h1:EYyT39nXdZDNVmgbJAlyIVWbEb1txnAOKpJPSYpvgXk=
-github.com/sourcegraph/sourcegraph/lib v0.0.0-20220825181731-397a768a5290 h1:SLCu3Rf1eLZ4sNKl0Bg1oURTgDxEutRCaTQt5dpVqH4=
-github.com/sourcegraph/sourcegraph/lib v0.0.0-20220825181731-397a768a5290/go.mod h1:9wnFUNfpORLAOJn4XAO7ZeWnYkf6/CxlWaTU1vlpuKc=
+github.com/sourcegraph/sourcegraph/lib v0.0.0-20221004162410-237d5855fa13 h1:HEj9QVz35nIU4plPb7iJSFjcTA14zVnfAdfYZF5jsKg=
+github.com/sourcegraph/sourcegraph/lib v0.0.0-20221004162410-237d5855fa13/go.mod h1:e3v3msUOgthv/QCWUzhFdfR3+hwUSn37rtuw49IVJg4=
 github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152 h1:z/MpntplPaW6QW95pzcAR/72Z5TWDyDnSo0EOcyij9o=
 github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -540,8 +540,8 @@ golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
+golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8=
 golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/batches/executor/coordinator.go

@@ -67,7 +67,7 @@ func (c *Coordinator) CheckCache(ctx context.Context, batchSpec *batcheslib.Batc
 func (c *Coordinator) ClearCache(ctx context.Context, tasks []*Task) error {
 	for _, task := range tasks {
 		for i := len(task.Steps) - 1; i > -1; i-- {
-			key := task.CacheKey(c.opts.GlobalEnv, c.opts.IsRemote, i)
+			key := task.CacheKey(c.opts.GlobalEnv, c.opts.ExecOpts.WorkingDirectory, i)
 			if err := c.opts.Cache.Clear(ctx, key); err != nil {
 				return errors.Wrapf(err, "clearing cache for step %d in %q", i, task.Repository.Name)
 			}
@@ -100,7 +100,7 @@ func (c *Coordinator) checkCacheForTask(ctx context.Context, batchSpec *batchesl
 	return specs, false, nil
 }
 
-func (c Coordinator) buildChangesetSpecs(task *Task, batchSpec *batcheslib.BatchSpec, result execution.AfterStepResult) ([]*batcheslib.ChangesetSpec, error) {
+func (c *Coordinator) buildChangesetSpecs(task *Task, batchSpec *batcheslib.BatchSpec, result execution.AfterStepResult) ([]*batcheslib.ChangesetSpec, error) {
 	input := &batcheslib.ChangesetSpecInput{
 		Repository: batcheslib.Repository{
 			ID:          task.Repository.ID,
@@ -128,7 +128,7 @@ func (c *Coordinator) loadCachedStepResults(ctx context.Context, task *Task, glo
 	// We start at the back so that we can find the _last_ cached step,
 	// then restart execution on the following step.
 	for i := len(task.Steps) - 1; i > -1; i-- {
-		key := task.CacheKey(globalEnv, c.opts.IsRemote, i)
+		key := task.CacheKey(globalEnv, c.opts.ExecOpts.WorkingDirectory, i)
 
 		result, found, err := c.opts.Cache.Get(ctx, key)
 		if err != nil {
@@ -181,7 +181,7 @@ func (c *Coordinator) ExecuteAndBuildSpecs(ctx context.Context, batchSpec *batch
 	// Write all step cache results to the cache.
 	for _, res := range results {
 		for _, stepRes := range res.stepResults {
-			cacheKey := res.task.CacheKey(c.opts.GlobalEnv, c.opts.IsRemote, stepRes.StepIndex)
+			cacheKey := res.task.CacheKey(c.opts.GlobalEnv, c.opts.ExecOpts.WorkingDirectory, stepRes.StepIndex)
 			if err := c.opts.Cache.Set(ctx, cacheKey, stepRes); err != nil {
 				return nil, nil, errors.Wrapf(err, "caching result for step %d", stepRes.StepIndex)
 			}

internal/batches/executor/executor.go

@@ -62,11 +62,12 @@ type NewExecutorOpts struct {
 	Logger              log.LogManager
 
 	// Config
-	Parallelism int
-	Timeout     time.Duration
-	TempDir     string
-	IsRemote    bool
-	GlobalEnv   []string
+	Parallelism      int
+	Timeout          time.Duration
+	WorkingDirectory string
+	TempDir          string
+	IsRemote         bool
+	GlobalEnv        []string
 }
 
 type executor struct {
@@ -168,15 +169,15 @@ func (x *executor) do(ctx context.Context, task *Task, ui TaskExecutionUI) (err
 
 	// Actually execute the steps.
 	opts := &RunStepsOpts{
-		Task:        task,
-		Logger:      l,
-		WC:          x.opts.Creator,
-		EnsureImage: x.opts.EnsureImage,
-		TempDir:     x.opts.TempDir,
-		IsRemote:    x.opts.IsRemote,
-		GlobalEnv:   x.opts.GlobalEnv,
-		Timeout:     x.opts.Timeout,
-		RepoArchive: repoArchive,
+		Task:             task,
+		Logger:           l,
+		WC:               x.opts.Creator,
+		EnsureImage:      x.opts.EnsureImage,
+		TempDir:          x.opts.TempDir,
+		GlobalEnv:        x.opts.GlobalEnv,
+		Timeout:          x.opts.Timeout,
+		RepoArchive:      repoArchive,
+		WorkingDirectory: x.opts.WorkingDirectory,
 
 		UI: ui.StepsExecutionUI(task),
 	}