diff --git a/pkg/build/local/run_executor.go b/pkg/build/local/run_executor.go index a0156d8d..c13a3bc9 100644 --- a/pkg/build/local/run_executor.go +++ b/pkg/build/local/run_executor.go @@ -38,6 +38,7 @@ type DockerRunExecutor struct { tempDirBase string authCallback AuthCallback allowPrivileged bool + memoryLimit string } // NewDockerRunExecutor creates a new Docker run executor with configuration @@ -81,6 +82,7 @@ func NewDockerRunExecutor(config DockerRunExecutorConfig) (*DockerRunExecutor, e tempDirBase: tempBase, authCallback: config.AuthCallback, allowPrivileged: config.AllowPrivileged, + memoryLimit: config.MemoryLimit, }, nil } @@ -98,6 +100,7 @@ type DockerRunExecutorConfig struct { TempDirBase string // Base directory for temp files, if empty uses os.TempDir() AuthCallback AuthCallback // Optional callback to generate auth headers when needed AllowPrivileged bool // If true, allow privileged builds + MemoryLimit string // If provided, sets a memory limit on the container (ex 512m or 1g) } // Start implements build.Executor @@ -213,6 +216,9 @@ func (e *DockerRunExecutor) executeBuild(ctx context.Context, handle *localHandl log.Println("Warning: plan requested privileged execution but this executor does not allow privileged builds.") } } + if e.memoryLimit != "" { + runArgs = append(runArgs, "--memory", e.memoryLimit) + } // Add AUTH_HEADER environment variable if auth is required and callback is available if plan.RequiresAuth && e.authCallback != nil { diff --git a/tools/benchmark/run/local.go b/tools/benchmark/run/local.go index c840d174..e73ed966 100644 --- a/tools/benchmark/run/local.go +++ b/tools/benchmark/run/local.go @@ -30,21 +30,23 @@ import ( ) type localExecutionService struct { - prebuildURL string - store rebuild.LocatableAssetStore - logsink io.Writer - gitCache *gitcache.Client + prebuildURL string + store rebuild.LocatableAssetStore + logsink io.Writer + gitCache *gitcache.Client + dockerConfig local.DockerRunExecutorConfig } type LocalExecutionServiceConfig struct { - PrebuildURL string - Store rebuild.LocatableAssetStore - LogSink io.Writer - GitCache *gitcache.Client + PrebuildURL string + Store rebuild.LocatableAssetStore + LogSink io.Writer + GitCache *gitcache.Client + DockerConfig local.DockerRunExecutorConfig } func NewLocalExecutionService(config LocalExecutionServiceConfig) ExecutionService { - return &localExecutionService{prebuildURL: config.PrebuildURL, store: config.Store, logsink: config.LogSink, gitCache: config.GitCache} + return &localExecutionService{prebuildURL: config.PrebuildURL, store: config.Store, logsink: config.LogSink, gitCache: config.GitCache, dockerConfig: config.DockerConfig} } func (s *localExecutionService) RebuildPackage(ctx context.Context, req schema.RebuildPackageRequest) (*schema.Verdict, error) { @@ -73,7 +75,7 @@ func (s *localExecutionService) RebuildPackage(ctx context.Context, req schema.R return verdict, nil } verdict.StrategyOneof = schema.NewStrategyOneOf(strategy) - if err := executeBuild(ctx, t, strategy, s.store, buildOpts{PrebuildURL: s.prebuildURL, LogSink: s.logsink}); err != nil { + if err := executeBuild(ctx, t, strategy, s.store, buildOpts{PrebuildURL: s.prebuildURL, LogSink: s.logsink}, s.dockerConfig); err != nil { verdict.Message = err.Error() } else if err := compare(ctx, t, s.store, mux); err != nil { verdict.Message = err.Error() @@ -98,7 +100,7 @@ func RebuildWithStrategy(ctx context.Context, executor ExecutionService, t rebui } verdict := &schema.Verdict{Target: t} verdict.StrategyOneof = schema.NewStrategyOneOf(strategy) - if err := executeBuild(ctx, t, strategy, local.store, buildOpts{PrebuildURL: local.prebuildURL, LogSink: local.logsink}); err != nil { + if err := executeBuild(ctx, t, strategy, local.store, buildOpts{PrebuildURL: local.prebuildURL, LogSink: local.logsink}, local.dockerConfig); err != nil { verdict.Message = err.Error() } else if err := compare(ctx, t, local.store, mux); err != nil { verdict.Message = err.Error() @@ -151,11 +153,8 @@ type buildOpts struct { LogSink io.Writer } -func executeBuild(ctx context.Context, t rebuild.Target, strategy rebuild.Strategy, out rebuild.LocatableAssetStore, opts buildOpts) error { - executor, err := local.NewDockerRunExecutor(local.DockerRunExecutorConfig{ - Planner: local.NewDockerRunPlanner(), - MaxParallel: 1, - }) +func executeBuild(ctx context.Context, t rebuild.Target, strategy rebuild.Strategy, out rebuild.LocatableAssetStore, opts buildOpts, config local.DockerRunExecutorConfig) error { + executor, err := local.NewDockerRunExecutor(config) if err != nil { return errors.Wrap(err, "failed to create executor") } diff --git a/tools/ctl/command/runbenchmark/runbenchmark.go b/tools/ctl/command/runbenchmark/runbenchmark.go index 84ece006..7a7c1bb7 100644 --- a/tools/ctl/command/runbenchmark/runbenchmark.go +++ b/tools/ctl/command/runbenchmark/runbenchmark.go @@ -26,6 +26,7 @@ import ( "github.com/google/oss-rebuild/internal/taskqueue" "github.com/google/oss-rebuild/pkg/act" "github.com/google/oss-rebuild/pkg/act/cli" + "github.com/google/oss-rebuild/pkg/build/local" "github.com/google/oss-rebuild/pkg/rebuild/schema" "github.com/google/oss-rebuild/tools/benchmark" benchrun "github.com/google/oss-rebuild/tools/benchmark/run" @@ -39,6 +40,7 @@ import ( type Config struct { API string Local bool + MemoryLimit string BenchmarkPath string BootstrapBucket string BootstrapVersion string @@ -66,6 +68,9 @@ func (c Config) Validate() error { if c.Format != "" && c.Format != "summary" && c.Format != "csv" { return errors.Errorf("invalid format: %s. Expected one of 'summary' or 'csv'", c.Format) } + if !c.Local && c.MemoryLimit != "" { + return errors.New("memory is only supported in local mode") + } if !c.Local && c.GitCacheURL != "" { return errors.New("git-cache-url is only supported in local mode") } @@ -137,6 +142,10 @@ func Handler(ctx context.Context, cfg Config, deps *Deps) (*act.NoOutput, error) PrebuildURL: prebuildURL, Store: store, LogSink: deps.IO.Out, + DockerConfig: local.DockerRunExecutorConfig{ + MaxParallel: 1, + MemoryLimit: cfg.MemoryLimit, + }, } if cfg.GitCacheURL != "" { u, err := url.Parse(cfg.GitCacheURL) @@ -292,6 +301,7 @@ func flagSet(name string, cfg *Config) *flag.FlagSet { set.StringVar(&cfg.API, "api", "", "OSS Rebuild API endpoint URI") set.IntVar(&cfg.MaxConcurrency, "max-concurrency", 90, "maximum number of inflight requests") set.BoolVar(&cfg.Local, "local", false, "true if this request is going direct to build-local (not through API first)") + set.StringVar(&cfg.MemoryLimit, "memory", "", "memory limit to be passed to docker (local mode only)") set.StringVar(&cfg.BootstrapBucket, "bootstrap-bucket", "", "the gcs bucket where bootstrap tools are stored") set.StringVar(&cfg.BootstrapVersion, "bootstrap-version", "", "the version of bootstrap tools to use") set.StringVar(&cfg.Format, "format", "", "format of the output (summary|csv)") diff --git a/tools/ctl/command/runone/runone.go b/tools/ctl/command/runone/runone.go index de4043fb..42586df4 100644 --- a/tools/ctl/command/runone/runone.go +++ b/tools/ctl/command/runone/runone.go @@ -19,6 +19,7 @@ import ( "github.com/google/oss-rebuild/internal/oauth" "github.com/google/oss-rebuild/pkg/act" "github.com/google/oss-rebuild/pkg/act/cli" + "github.com/google/oss-rebuild/pkg/build/local" "github.com/google/oss-rebuild/pkg/rebuild/rebuild" "github.com/google/oss-rebuild/pkg/rebuild/schema" benchrun "github.com/google/oss-rebuild/tools/benchmark/run" @@ -34,6 +35,7 @@ const analyzeMode = schema.ExecutionMode("analyze") type Config struct { API string Local bool + MemoryLimit string BootstrapBucket string BootstrapVersion string GitCacheURL string @@ -75,6 +77,9 @@ func (c Config) Validate() error { if c.Local && mode == analyzeMode { return errors.New("analyze mode is not supported in local execution") } + if !c.Local && c.MemoryLimit != "" { + return errors.New("memory is only supported in local mode") + } if !c.Local && c.GitCacheURL != "" { return errors.New("git-cache-url is only supported in local mode") } @@ -150,6 +155,10 @@ func handleLocal(ctx context.Context, cfg Config, deps *Deps, enc *json.Encoder, PrebuildURL: prebuildURL, Store: store, LogSink: deps.IO.Out, + DockerConfig: local.DockerRunExecutorConfig{ + MaxParallel: 1, + MemoryLimit: cfg.MemoryLimit, + }, } if cfg.GitCacheURL != "" { u, err := url.Parse(cfg.GitCacheURL) @@ -280,6 +289,7 @@ func flagSet(name string, cfg *Config) *flag.FlagSet { set := flag.NewFlagSet(name, flag.ContinueOnError) set.StringVar(&cfg.API, "api", "", "OSS Rebuild API endpoint URI") set.BoolVar(&cfg.Local, "local", false, "run locally instead of through the API") + set.StringVar(&cfg.MemoryLimit, "memory", "", "memory limit to be passed to docker (local mode only)") set.StringVar(&cfg.BootstrapBucket, "bootstrap-bucket", "", "the GCS bucket where bootstrap tools are stored (required for local mode)") set.StringVar(&cfg.BootstrapVersion, "bootstrap-version", "", "the version of bootstrap tools to use (required for local mode)") set.StringVar(&cfg.GitCacheURL, "git-cache-url", "", "if provided, the git-cache service to use to fetch repos (local mode only)")