Skip to content

Add memory limit flag to runone and runbench #1096

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wbxyz wants to merge 1 commit into
base: main
Choose a base branch
from
+41 −16
Open

Add memory limit flag to runone and runbench #1096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkg/build/local/run_executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type DockerRunExecutor struct {
tempDirBase string
authCallback AuthCallback
allowPrivileged bool
memoryLimit string
}

// NewDockerRunExecutor creates a new Docker run executor with configuration
Expand Down Expand Up @@ -81,6 +82,7 @@ func NewDockerRunExecutor(config DockerRunExecutorConfig) (*DockerRunExecutor, e
tempDirBase: tempBase,
authCallback: config.AuthCallback,
allowPrivileged: config.AllowPrivileged,
memoryLimit: config.MemoryLimit,
}, nil
}

Expand All @@ -98,6 +100,7 @@ type DockerRunExecutorConfig struct {
TempDirBase string // Base directory for temp files, if empty uses os.TempDir()
AuthCallback AuthCallback // Optional callback to generate auth headers when needed
AllowPrivileged bool // If true, allow privileged builds
MemoryLimit string // If provided, sets a memory limit on the container (ex 512m or 1g)
}

// Start implements build.Executor
Expand Down Expand Up @@ -213,6 +216,9 @@ func (e *DockerRunExecutor) executeBuild(ctx context.Context, handle *localHandl
log.Println("Warning: plan requested privileged execution but this executor does not allow privileged builds.")
}
}
if e.memoryLimit != "" {
runArgs = append(runArgs, "--memory", e.memoryLimit)
}

// Add AUTH_HEADER environment variable if auth is required and callback is available
if plan.RequiresAuth && e.authCallback != nil {
Expand Down
31 changes: 15 additions & 16 deletions tools/benchmark/run/local.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,23 @@ import (
)

type localExecutionService struct {
prebuildURL string
store rebuild.LocatableAssetStore
logsink io.Writer
gitCache *gitcache.Client
prebuildURL string
store rebuild.LocatableAssetStore
logsink io.Writer
gitCache *gitcache.Client
dockerConfig local.DockerRunExecutorConfig
}

type LocalExecutionServiceConfig struct {
PrebuildURL string
Store rebuild.LocatableAssetStore
LogSink io.Writer
GitCache *gitcache.Client
PrebuildURL string
Store rebuild.LocatableAssetStore
LogSink io.Writer
GitCache *gitcache.Client
DockerConfig local.DockerRunExecutorConfig
}

func NewLocalExecutionService(config LocalExecutionServiceConfig) ExecutionService {
return &localExecutionService{prebuildURL: config.PrebuildURL, store: config.Store, logsink: config.LogSink, gitCache: config.GitCache}
return &localExecutionService{prebuildURL: config.PrebuildURL, store: config.Store, logsink: config.LogSink, gitCache: config.GitCache, dockerConfig: config.DockerConfig}
}

func (s *localExecutionService) RebuildPackage(ctx context.Context, req schema.RebuildPackageRequest) (*schema.Verdict, error) {
Expand Down Expand Up @@ -73,7 +75,7 @@ func (s *localExecutionService) RebuildPackage(ctx context.Context, req schema.R
return verdict, nil
}
verdict.StrategyOneof = schema.NewStrategyOneOf(strategy)
if err := executeBuild(ctx, t, strategy, s.store, buildOpts{PrebuildURL: s.prebuildURL, LogSink: s.logsink}); err != nil {
if err := executeBuild(ctx, t, strategy, s.store, buildOpts{PrebuildURL: s.prebuildURL, LogSink: s.logsink}, s.dockerConfig); err != nil {
verdict.Message = err.Error()
} else if err := compare(ctx, t, s.store, mux); err != nil {
verdict.Message = err.Error()
Expand All @@ -98,7 +100,7 @@ func RebuildWithStrategy(ctx context.Context, executor ExecutionService, t rebui
}
verdict := &schema.Verdict{Target: t}
verdict.StrategyOneof = schema.NewStrategyOneOf(strategy)
if err := executeBuild(ctx, t, strategy, local.store, buildOpts{PrebuildURL: local.prebuildURL, LogSink: local.logsink}); err != nil {
if err := executeBuild(ctx, t, strategy, local.store, buildOpts{PrebuildURL: local.prebuildURL, LogSink: local.logsink}, local.dockerConfig); err != nil {
verdict.Message = err.Error()
} else if err := compare(ctx, t, local.store, mux); err != nil {
verdict.Message = err.Error()
Expand Down Expand Up @@ -151,11 +153,8 @@ type buildOpts struct {
LogSink io.Writer
}

func executeBuild(ctx context.Context, t rebuild.Target, strategy rebuild.Strategy, out rebuild.LocatableAssetStore, opts buildOpts) error {
executor, err := local.NewDockerRunExecutor(local.DockerRunExecutorConfig{
Planner: local.NewDockerRunPlanner(),
MaxParallel: 1,
})
func executeBuild(ctx context.Context, t rebuild.Target, strategy rebuild.Strategy, out rebuild.LocatableAssetStore, opts buildOpts, config local.DockerRunExecutorConfig) error {
executor, err := local.NewDockerRunExecutor(config)
if err != nil {
return errors.Wrap(err, "failed to create executor")
}
Expand Down
10 changes: 10 additions & 0 deletions tools/ctl/command/runbenchmark/runbenchmark.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/google/oss-rebuild/internal/taskqueue"
"github.com/google/oss-rebuild/pkg/act"
"github.com/google/oss-rebuild/pkg/act/cli"
"github.com/google/oss-rebuild/pkg/build/local"
"github.com/google/oss-rebuild/pkg/rebuild/schema"
"github.com/google/oss-rebuild/tools/benchmark"
benchrun "github.com/google/oss-rebuild/tools/benchmark/run"
Expand All @@ -39,6 +40,7 @@ import (
type Config struct {
API string
Local bool
MemoryLimit string
BenchmarkPath string
BootstrapBucket string
BootstrapVersion string
Expand Down Expand Up @@ -66,6 +68,9 @@ func (c Config) Validate() error {
if c.Format != "" && c.Format != "summary" && c.Format != "csv" {
return errors.Errorf("invalid format: %s. Expected one of 'summary' or 'csv'", c.Format)
}
if !c.Local && c.MemoryLimit != "" {
return errors.New("memory is only supported in local mode")
}
if !c.Local && c.GitCacheURL != "" {
return errors.New("git-cache-url is only supported in local mode")
}
Expand Down Expand Up @@ -137,6 +142,10 @@ func Handler(ctx context.Context, cfg Config, deps *Deps) (*act.NoOutput, error)
PrebuildURL: prebuildURL,
Store: store,
LogSink: deps.IO.Out,
DockerConfig: local.DockerRunExecutorConfig{
MaxParallel: 1,
MemoryLimit: cfg.MemoryLimit,
},
}
if cfg.GitCacheURL != "" {
u, err := url.Parse(cfg.GitCacheURL)
Expand Down Expand Up @@ -292,6 +301,7 @@ func flagSet(name string, cfg *Config) *flag.FlagSet {
set.StringVar(&cfg.API, "api", "", "OSS Rebuild API endpoint URI")
set.IntVar(&cfg.MaxConcurrency, "max-concurrency", 90, "maximum number of inflight requests")
set.BoolVar(&cfg.Local, "local", false, "true if this request is going direct to build-local (not through API first)")
set.StringVar(&cfg.MemoryLimit, "memory", "", "memory limit to be passed to docker (local mode only)")
set.StringVar(&cfg.BootstrapBucket, "bootstrap-bucket", "", "the gcs bucket where bootstrap tools are stored")
set.StringVar(&cfg.BootstrapVersion, "bootstrap-version", "", "the version of bootstrap tools to use")
set.StringVar(&cfg.Format, "format", "", "format of the output (summary|csv)")
Expand Down
10 changes: 10 additions & 0 deletions tools/ctl/command/runone/runone.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/google/oss-rebuild/internal/oauth"
"github.com/google/oss-rebuild/pkg/act"
"github.com/google/oss-rebuild/pkg/act/cli"
"github.com/google/oss-rebuild/pkg/build/local"
"github.com/google/oss-rebuild/pkg/rebuild/rebuild"
"github.com/google/oss-rebuild/pkg/rebuild/schema"
benchrun "github.com/google/oss-rebuild/tools/benchmark/run"
Expand All @@ -34,6 +35,7 @@ const analyzeMode = schema.ExecutionMode("analyze")
type Config struct {
API string
Local bool
MemoryLimit string
BootstrapBucket string
BootstrapVersion string
GitCacheURL string
Expand Down Expand Up @@ -75,6 +77,9 @@ func (c Config) Validate() error {
if c.Local && mode == analyzeMode {
return errors.New("analyze mode is not supported in local execution")
}
if !c.Local && c.MemoryLimit != "" {
return errors.New("memory is only supported in local mode")
}
if !c.Local && c.GitCacheURL != "" {
return errors.New("git-cache-url is only supported in local mode")
}
Expand Down Expand Up @@ -150,6 +155,10 @@ func handleLocal(ctx context.Context, cfg Config, deps *Deps, enc *json.Encoder,
PrebuildURL: prebuildURL,
Store: store,
LogSink: deps.IO.Out,
DockerConfig: local.DockerRunExecutorConfig{
MaxParallel: 1,
MemoryLimit: cfg.MemoryLimit,
},
}
if cfg.GitCacheURL != "" {
u, err := url.Parse(cfg.GitCacheURL)
Expand Down Expand Up @@ -280,6 +289,7 @@ func flagSet(name string, cfg *Config) *flag.FlagSet {
set := flag.NewFlagSet(name, flag.ContinueOnError)
set.StringVar(&cfg.API, "api", "", "OSS Rebuild API endpoint URI")
set.BoolVar(&cfg.Local, "local", false, "run locally instead of through the API")
set.StringVar(&cfg.MemoryLimit, "memory", "", "memory limit to be passed to docker (local mode only)")
set.StringVar(&cfg.BootstrapBucket, "bootstrap-bucket", "", "the GCS bucket where bootstrap tools are stored (required for local mode)")
set.StringVar(&cfg.BootstrapVersion, "bootstrap-version", "", "the version of bootstrap tools to use (required for local mode)")
set.StringVar(&cfg.GitCacheURL, "git-cache-url", "", "if provided, the git-cache service to use to fetch repos (local mode only)")
Expand Down
Loading