From b22b7caba18ea2d1740e5275b1355511ba7e638b Mon Sep 17 00:00:00 2001 From: Mark Sumner Date: Tue, 19 Sep 2023 11:09:30 +0000 Subject: [PATCH] Wire in min cardinality (#8) * Wire up minimum gang cardinality * Wire up minimum gang cardinality * Wire up minimum gang cardinality * Wire up minimum gang cardinality * Bump armada airflow operator to version 0.5.4 (#2961) * Bump armada airflow operator to version 0.5.4 Signed-off-by: Rich Scott * Regenerate Airflow Operator Markdown doc. Signed-off-by: Rich Scott * Fix regenerated Airflow doc error. Signed-off-by: Rich Scott * Pin versions of all modules, especially around docs generation. Signed-off-by: Rich Scott * Regenerate Airflow docs using Python 3.10 Signed-off-by: Rich Scott --------- Signed-off-by: Rich Scott * Infer failed jobs from job context, tidy up * Infer failed jobs from job context, tidy up * Magefile: Clean all Makefile refernces (#2957) * tiny naming change * clean all make refernces Signed-off-by: mohamed --------- Signed-off-by: mohamed * Infer failed jobs from job context, tidy up * Revert to previous unpinned airflow version spec. (#2967) * Revert to previous unpinned airflow version spec. Signed-off-by: Rich Scott * Increment armada-airflow module version. Signed-off-by: Rich Scott --------- Signed-off-by: Rich Scott * Only fail gang jobs when the overall gang min cardinality is set. Fix error handling * Only fail gang jobs when the overall gang min cardinality is set. Fix error handling * Only fail gang jobs when the overall gang min cardinality is set. Fix error handling * Update jobdb with any excess gang jobs that failed * ArmadaContext.Log Improvements (#2965) * log error * context log * context log * add cycle id * typo * lint * refactor armadacontext to implement a FieldLogger --------- Co-authored-by: Chris Martin * Fix-up existing tests before adding new ones * Add new tests for minimum gang sizes * Test that excess failed gang jobs are committed to jobdb * Run `on.push` only for master (#2968) * Run On Push only for master Signed-off-by: mohamed * remove not-workflows Signed-off-by: mohamed --------- Signed-off-by: mohamed * Add test for failed job pulsar messages * Tidy tests * WIP: Airflow: fix undefined poll_interval in Deferrable Operator (#2975) * Airflow: handle poll_interval attr in ArmadaJobCompleteTrigger Fix incomplete handling of 'poll_interval' attribute in ArmadaJobCompleteTrigger, used by the Armada Deferrable Operator for Airflow. Signed-off-by: Rich Scott * Airflow - add unit test for armada deferrable operator Run much of the same tests for the deferrable operator as for the regular operator, plus test serialization. Also, update interval signifier in examples. A full test of the deferrable operator that verifies the trigger handling is still needed. Signed-off-by: Rich Scott --------- Signed-off-by: Rich Scott * Release Airflow Operator v0.5.6 (#2979) Signed-off-by: Rich Scott * #2905 - fix indentation (#2971) Co-authored-by: Mohamed Abdelfatah <39927413+Mo-Fatah@users.noreply.github.com> Co-authored-by: Adam McArthur <46480158+Sharpz7@users.noreply.github.com> Signed-off-by: Rich Scott Signed-off-by: mohamed Co-authored-by: Rich Scott Co-authored-by: Mohamed Abdelfatah <39927413+Mo-Fatah@users.noreply.github.com> Co-authored-by: Chris Martin Co-authored-by: Chris Martin Co-authored-by: Dave Gantenbein Co-authored-by: Adam McArthur <46480158+Sharpz7@users.noreply.github.com> --- .github/workflows/autoupdate.yml | 2 +- .github/workflows/ci.yml | 4 +- .github/workflows/not-airflow-operator.yml | 47 -- .github/workflows/not-python-client.yml | 42 -- .github/workflows/test.yml | 2 +- client/python/CONTRIBUTING.md | 2 +- client/python/README.md | 2 +- client/python/docs/README.md | 4 +- cmd/scheduler/cmd/prune_database.go | 4 +- docs/developer/manual-localdev.md | 2 +- docs/python_airflow_operator.md | 30 +- internal/armada/server/lease.go | 12 +- internal/armada/server/submit_from_log.go | 8 +- .../common/armadacontext/armada_context.go | 38 +- .../armadacontext/armada_context_test.go | 8 +- internal/common/logging/stacktrace.go | 6 +- internal/scheduler/api.go | 4 +- internal/scheduler/common.go | 35 +- internal/scheduler/context/context.go | 47 +- internal/scheduler/context/context_test.go | 7 +- internal/scheduler/database/db_pruner.go | 15 +- internal/scheduler/database/util.go | 3 +- internal/scheduler/gang_scheduler.go | 126 ++-- internal/scheduler/gang_scheduler_test.go | 183 +++-- internal/scheduler/leader.go | 8 +- internal/scheduler/metrics.go | 18 +- internal/scheduler/nodedb/nodedb.go | 50 +- internal/scheduler/nodedb/nodedb_test.go | 53 +- internal/scheduler/pool_assigner.go | 9 +- .../scheduler/preempting_queue_scheduler.go | 15 +- .../preempting_queue_scheduler_test.go | 2 +- internal/scheduler/publisher.go | 10 +- internal/scheduler/queue_scheduler.go | 17 +- internal/scheduler/queue_scheduler_test.go | 8 +- internal/scheduler/reports_test.go | 4 +- internal/scheduler/scheduler.go | 83 ++- internal/scheduler/scheduler_metrics.go | 34 +- internal/scheduler/scheduler_test.go | 26 +- internal/scheduler/schedulerapp.go | 32 +- internal/scheduler/scheduling_algo.go | 27 +- internal/scheduler/scheduling_algo_test.go | 28 +- internal/scheduler/simulator/simulator.go | 4 + internal/scheduler/submitcheck.go | 24 +- internal/scheduler/submitcheck_test.go | 6 +- .../scheduler/testfixtures/testfixtures.go | 12 +- magefiles/linting.go | 2 +- pkg/armadaevents/events.pb.go | 693 ++++++++++++------ pkg/armadaevents/events.proto | 5 + .../armada/operators/armada_deferrable.py | 39 +- third_party/airflow/armada/operators/utils.py | 4 +- third_party/airflow/examples/big_armada.py | 2 +- third_party/airflow/pyproject.toml | 10 +- .../tests/unit/test_airflow_operator_mock.py | 4 +- .../unit/test_armada_deferrable_operator.py | 171 +++++ .../test_search_for_job_complete_asyncio.py | 5 + 55 files changed, 1394 insertions(+), 644 deletions(-) delete mode 100644 .github/workflows/not-airflow-operator.yml delete mode 100644 .github/workflows/not-python-client.yml create mode 100644 third_party/airflow/tests/unit/test_armada_deferrable_operator.py diff --git a/.github/workflows/autoupdate.yml b/.github/workflows/autoupdate.yml index de45e651c8e..425f72538ec 100644 --- a/.github/workflows/autoupdate.yml +++ b/.github/workflows/autoupdate.yml @@ -15,7 +15,7 @@ jobs: - uses: docker://chinthakagodawita/autoupdate-action:v1 env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' - PR_LABELS: "auto-update" + PR_LABELS: "auto-update" MERGE_MSG: "Branch was auto-updated." RETRY_COUNT: "5" RETRY_SLEEP: "300" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 125b18d3096..1b688b3731b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,10 @@ name: CI on: push: + branches: + - master tags: - v* - branches-ignore: - - gh-pages pull_request: branches-ignore: - gh-pages diff --git a/.github/workflows/not-airflow-operator.yml b/.github/workflows/not-airflow-operator.yml deleted file mode 100644 index 298cb79c0fd..00000000000 --- a/.github/workflows/not-airflow-operator.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Python Airflow Operator - -on: - push: - branches-ignore: - - master - paths-ignore: - - 'client/python/**' - - 'build/python-client/**' - - 'pkg/api/*.proto' - - '.github/workflows/python-client.yml' - - 'docs/python_armada_client.md' - - 'scripts/build-python-client.sh' - - 'third_party/airflow/**' - - 'build/airflow-operator/**' - - 'pkg/api/jobservice/*.proto' - - '.github/workflows/airflow-operator.yml' - - 'docs/python_airflow_operator.md' - - 'scripts/build-airflow-operator.sh' - - '.github/workflows/python-tests/*' - - pull_request: - branches-ignore: - - gh-pages - paths-ignore: - - 'client/python/**' - - 'build/python-client/**' - - 'pkg/api/*.proto' - - '.github/workflows/python-client.yml' - - 'docs/python_armada_client.md' - - 'scripts/build-python-client.sh' - - 'third_party/airflow/**' - - 'build/airflow-operator/**' - - 'pkg/api/jobservice/*.proto' - - '.github/workflows/airflow-operator.yml' - - 'docs/python_airflow_operator.md' - - 'scripts/build-airflow-operator.sh' - - '.github/workflows/python-tests/*' - -jobs: - airflow-tox: - strategy: - matrix: - go: [ '1.20' ] - runs-on: ubuntu-latest - steps: - - run: 'echo "No airflow operator code modified, not running airflow operator jobs"' diff --git a/.github/workflows/not-python-client.yml b/.github/workflows/not-python-client.yml deleted file mode 100644 index a7704606427..00000000000 --- a/.github/workflows/not-python-client.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Python Client - -on: - push: - branches-ignore: - - master - paths-ignore: - - 'client/python/**' - - 'build/python-client/**' - - 'pkg/api/*.proto' - - '.github/workflows/python-client.yml' - - 'docs/python_armada_client.md' - - 'scripts/build-python-client.sh' - - '.github/workflows/python-tests/*' - - pull_request: - branches-ignore: - - gh-pages - paths-ignore: - - 'client/python/**' - - 'build/python-client/**' - - 'pkg/api/*.proto' - - '.github/workflows/python-client.yml' - - 'docs/python_armada_client.md' - - 'scripts/build-python-client.sh' - - '.github/workflows/python-tests/*' - -jobs: - python-client-tox: - strategy: - matrix: - go: [ '1.20' ] - runs-on: ubuntu-latest - steps: - - run: 'echo "No python modified, not running python jobs"' - python-client-integration-tests: - strategy: - matrix: - go: [ '1.20' ] - runs-on: ubuntu-latest - steps: - - run: 'echo "No python modified, not running python jobs"' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b0ea22a381..887c3658836 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -214,7 +214,7 @@ jobs: echo -e "### Git status" >> $GITHUB_STEP_SUMMARY if [[ "$changed" -gt 0 ]]; then - echo -e "Generated proto files are out of date. Please run 'make proto' and commit the changes." >> $GITHUB_STEP_SUMMARY + echo -e "Generated proto files are out of date. Please run 'mage proto' and commit the changes." >> $GITHUB_STEP_SUMMARY git status -s -uno >> $GITHUB_STEP_SUMMARY diff --git a/client/python/CONTRIBUTING.md b/client/python/CONTRIBUTING.md index ca3c0f1f90d..ff015d4284e 100644 --- a/client/python/CONTRIBUTING.md +++ b/client/python/CONTRIBUTING.md @@ -26,7 +26,7 @@ workflow for contributing. First time contributors can follow the guide below to Unlike most python projects, the Armada python client contains a large quantity of generated code. This code must be generated in order to compile and develop against the client. -From the root of the repository, run `make python`. This will generate python code needed to build +From the root of the repository, run `mage buildPython`. This will generate python code needed to build and use the client. This command needs to be re-run anytime an API change is committed (e.g. a change to a `*.proto` file). diff --git a/client/python/README.md b/client/python/README.md index 92ed96b26b8..ea4f1409fb2 100644 --- a/client/python/README.md +++ b/client/python/README.md @@ -26,5 +26,5 @@ Before beginning, ensure you have: - Network access to fetch docker images and go dependencies. To generate all needed code, and install the python client: -1) From the root of the repository, run `make python` +1) From the root of the repository, run `mage buildPython` 2) Install the client using `pip install client/python`. It's strongly recommended you do this inside a virtualenv. diff --git a/client/python/docs/README.md b/client/python/docs/README.md index 056327c87ae..d8a7abfe1a0 100644 --- a/client/python/docs/README.md +++ b/client/python/docs/README.md @@ -9,13 +9,13 @@ Usage Easy way: - Ensure all protobufs files needed for the client are generated by running - `make python` from the repository root. + `mage buildPython` from the repository root. - `tox -e docs` will create a valid virtual environment and use it to generate documentation. The generated files will be placed under `build/jekyll/*.md`. Manual way: - Ensure all protobufs files needed for the client are generated by running - `make python` from the repository root. + `mage buildPython` from the repository root. - Create a virtual environment containing all the deps listed in `tox.ini` under `[testenv:docs]`. - Run `poetry install -v` from inside `client/python` to install the client diff --git a/cmd/scheduler/cmd/prune_database.go b/cmd/scheduler/cmd/prune_database.go index 3b2250d1661..4ed7aee426e 100644 --- a/cmd/scheduler/cmd/prune_database.go +++ b/cmd/scheduler/cmd/prune_database.go @@ -1,13 +1,13 @@ package cmd import ( - "context" "time" "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/clock" + "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/database" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" ) @@ -57,7 +57,7 @@ func pruneDatabase(cmd *cobra.Command, _ []string) error { return errors.WithMessagef(err, "Failed to connect to database") } - ctx, cancel := context.WithTimeout(context.Background(), timeout) + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), timeout) defer cancel() return schedulerdb.PruneDb(ctx, db, batchSize, expireAfter, clock.RealClock{}) } diff --git a/docs/developer/manual-localdev.md b/docs/developer/manual-localdev.md index 236995857c7..65c19e2faad 100644 --- a/docs/developer/manual-localdev.md +++ b/docs/developer/manual-localdev.md @@ -28,7 +28,7 @@ mage BootstrapTools # Compile .pb.go files from .proto files # (only necessary after changing a .proto file). mage proto -make dotnet +mage dotnet # Build the Docker images containing all Armada components. # Only the main "bundle" is needed for quickly testing Armada. diff --git a/docs/python_airflow_operator.md b/docs/python_airflow_operator.md index 1d820856344..048667a2562 100644 --- a/docs/python_airflow_operator.md +++ b/docs/python_airflow_operator.md @@ -239,9 +239,27 @@ Reports the result of the job and returns. +#### serialize() +Get a serialized version of this object. + + +* **Returns** + + A dict of keyword arguments used when instantiating + + + +* **Return type** + + dict + + +this object. + + #### template_fields(_: Sequence[str_ _ = ('job_request_items',_ ) -### _class_ armada.operators.armada_deferrable.ArmadaJobCompleteTrigger(job_id, job_service_channel_args, armada_queue, job_set_id, airflow_task_name) +### _class_ armada.operators.armada_deferrable.ArmadaJobCompleteTrigger(job_id, job_service_channel_args, armada_queue, job_set_id, airflow_task_name, poll_interval=30) Bases: `BaseTrigger` An airflow trigger that monitors the job state of an armada job. @@ -269,6 +287,9 @@ Triggers when the job is complete. belongs. + * **poll_interval** (*int*) – How often to poll jobservice to get status. + + * **Returns** @@ -281,7 +302,7 @@ Runs the trigger. Meant to be called by an airflow triggerer process. #### serialize() -Returns the information needed to reconstruct this Trigger. +Return the information needed to reconstruct this Trigger. * **Returns** @@ -664,7 +685,7 @@ A terminated event is SUCCEEDED, FAILED or CANCELLED -### _async_ armada.operators.utils.search_for_job_complete_async(armada_queue, job_set_id, airflow_task_name, job_id, job_service_client, log, time_out_for_failure=7200) +### _async_ armada.operators.utils.search_for_job_complete_async(armada_queue, job_set_id, airflow_task_name, job_id, job_service_client, log, poll_interval, time_out_for_failure=7200) Poll JobService cache asyncronously until you get a terminated event. A terminated event is SUCCEEDED, FAILED or CANCELLED @@ -689,6 +710,9 @@ A terminated event is SUCCEEDED, FAILED or CANCELLED It is optional only for testing + * **poll_interval** (*int*) – How often to poll jobservice to get status. + + * **time_out_for_failure** (*int*) – The amount of time a job can be in job_id_not_found before we decide it was a invalid job diff --git a/internal/armada/server/lease.go b/internal/armada/server/lease.go index 9a776d0e15f..2b9d7bb753e 100644 --- a/internal/armada/server/lease.go +++ b/internal/armada/server/lease.go @@ -344,7 +344,7 @@ func (q *AggregatedQueueServer) getJobs(ctx *armadacontext.Context, req *api.Str lastSeen, ) if err != nil { - logging.WithStacktrace(ctx.Log, err).Warnf( + logging.WithStacktrace(ctx, err).Warnf( "skipping node %s from executor %s", nodeInfo.GetName(), req.GetClusterId(), ) continue @@ -566,7 +566,7 @@ func (q *AggregatedQueueServer) getJobs(ctx *armadacontext.Context, req *api.Str if q.SchedulingContextRepository != nil { sctx.ClearJobSpecs() if err := q.SchedulingContextRepository.AddSchedulingContext(sctx); err != nil { - logging.WithStacktrace(ctx.Log, err).Error("failed to store scheduling context") + logging.WithStacktrace(ctx, err).Error("failed to store scheduling context") } } @@ -641,7 +641,7 @@ func (q *AggregatedQueueServer) getJobs(ctx *armadacontext.Context, req *api.Str jobIdsToDelete := util.Map(jobsToDelete, func(job *api.Job) string { return job.Id }) log.Infof("deleting preempted jobs: %v", jobIdsToDelete) if deletionResult, err := q.jobRepository.DeleteJobs(jobsToDelete); err != nil { - logging.WithStacktrace(ctx.Log, err).Error("failed to delete preempted jobs from Redis") + logging.WithStacktrace(ctx, err).Error("failed to delete preempted jobs from Redis") } else { deleteErrorByJobId := armadamaps.MapKeys(deletionResult, func(job *api.Job) string { return job.Id }) for jobId := range preemptedApiJobsById { @@ -704,7 +704,7 @@ func (q *AggregatedQueueServer) getJobs(ctx *armadacontext.Context, req *api.Str } } if err := q.usageRepository.UpdateClusterQueueResourceUsage(req.ClusterId, currentExecutorReport); err != nil { - logging.WithStacktrace(ctx.Log, err).Errorf("failed to update cluster usage") + logging.WithStacktrace(ctx, err).Errorf("failed to update cluster usage") } allocatedByQueueAndPriorityClassForPool = q.aggregateAllocationAcrossExecutor(reportsByExecutor, req.Pool) @@ -728,7 +728,7 @@ func (q *AggregatedQueueServer) getJobs(ctx *armadacontext.Context, req *api.Str } node, err := nodeDb.GetNode(nodeId) if err != nil { - logging.WithStacktrace(ctx.Log, err).Warnf("failed to set node id selector on job %s: node with id %s not found", apiJob.Id, nodeId) + logging.WithStacktrace(ctx, err).Warnf("failed to set node id selector on job %s: node with id %s not found", apiJob.Id, nodeId) continue } v := node.Labels[q.schedulingConfig.Preemption.NodeIdLabel] @@ -764,7 +764,7 @@ func (q *AggregatedQueueServer) getJobs(ctx *armadacontext.Context, req *api.Str } node, err := nodeDb.GetNode(nodeId) if err != nil { - logging.WithStacktrace(ctx.Log, err).Warnf("failed to set node name on job %s: node with id %s not found", apiJob.Id, nodeId) + logging.WithStacktrace(ctx, err).Warnf("failed to set node name on job %s: node with id %s not found", apiJob.Id, nodeId) continue } podSpec.NodeName = node.Name diff --git a/internal/armada/server/submit_from_log.go b/internal/armada/server/submit_from_log.go index 90b5ece3553..995e9785d5b 100644 --- a/internal/armada/server/submit_from_log.go +++ b/internal/armada/server/submit_from_log.go @@ -125,12 +125,12 @@ func (srv *SubmitFromLog) Run(ctx *armadacontext.Context) error { sequence, err := eventutil.UnmarshalEventSequence(ctxWithLogger, msg.Payload()) if err != nil { srv.ack(ctx, msg) - logging.WithStacktrace(ctxWithLogger.Log, err).Warnf("processing message failed; ignoring") + logging.WithStacktrace(ctxWithLogger, err).Warnf("processing message failed; ignoring") numErrored++ break } - ctxWithLogger.Log.WithField("numEvents", len(sequence.Events)).Info("processing sequence") + ctxWithLogger.WithField("numEvents", len(sequence.Events)).Info("processing sequence") // TODO: Improve retry logic. srv.ProcessSequence(ctxWithLogger, sequence) srv.ack(ctx, msg) @@ -155,11 +155,11 @@ func (srv *SubmitFromLog) ProcessSequence(ctx *armadacontext.Context, sequence * for i < len(sequence.Events) && time.Since(lastProgress) < timeout { j, err := srv.ProcessSubSequence(ctx, i, sequence) if err != nil { - logging.WithStacktrace(ctx.Log, err).WithFields(logrus.Fields{"lowerIndex": i, "upperIndex": j}).Warnf("processing subsequence failed; ignoring") + logging.WithStacktrace(ctx, err).WithFields(logrus.Fields{"lowerIndex": i, "upperIndex": j}).Warnf("processing subsequence failed; ignoring") } if j == i { - ctx.Log.WithFields(logrus.Fields{"lowerIndex": i, "upperIndex": j}).Info("made no progress") + ctx.WithFields(logrus.Fields{"lowerIndex": i, "upperIndex": j}).Info("made no progress") // We should only get here if a transient error occurs. // Sleep for a bit before retrying. diff --git a/internal/common/armadacontext/armada_context.go b/internal/common/armadacontext/armada_context.go index a6985ee5df7..0e41a66a1e4 100644 --- a/internal/common/armadacontext/armada_context.go +++ b/internal/common/armadacontext/armada_context.go @@ -13,22 +13,22 @@ import ( // while retaining type-safety type Context struct { context.Context - Log *logrus.Entry + logrus.FieldLogger } // Background creates an empty context with a default logger. It is analogous to context.Background() func Background() *Context { return &Context{ - Context: context.Background(), - Log: logrus.NewEntry(logrus.New()), + Context: context.Background(), + FieldLogger: logrus.NewEntry(logrus.New()), } } // TODO creates an empty context with a default logger. It is analogous to context.TODO() func TODO() *Context { return &Context{ - Context: context.TODO(), - Log: logrus.NewEntry(logrus.New()), + Context: context.TODO(), + FieldLogger: logrus.NewEntry(logrus.New()), } } @@ -42,8 +42,8 @@ func FromGrpcCtx(ctx context.Context) *Context { // New returns an armada context that encapsulates both a go context and a logger func New(ctx context.Context, log *logrus.Entry) *Context { return &Context{ - Context: ctx, - Log: log, + Context: ctx, + FieldLogger: log, } } @@ -51,8 +51,8 @@ func New(ctx context.Context, log *logrus.Entry) *Context { func WithCancel(parent *Context) (*Context, context.CancelFunc) { c, cancel := context.WithCancel(parent.Context) return &Context{ - Context: c, - Log: parent.Log, + Context: c, + FieldLogger: parent.FieldLogger, }, cancel } @@ -61,8 +61,8 @@ func WithCancel(parent *Context) (*Context, context.CancelFunc) { func WithDeadline(parent *Context, d time.Time) (*Context, context.CancelFunc) { c, cancel := context.WithDeadline(parent.Context, d) return &Context{ - Context: c, - Log: parent.Log, + Context: c, + FieldLogger: parent.FieldLogger, }, cancel } @@ -74,16 +74,16 @@ func WithTimeout(parent *Context, timeout time.Duration) (*Context, context.Canc // WithLogField returns a copy of parent with the supplied key-value added to the logger func WithLogField(parent *Context, key string, val interface{}) *Context { return &Context{ - Context: parent.Context, - Log: parent.Log.WithField(key, val), + Context: parent.Context, + FieldLogger: parent.FieldLogger.WithField(key, val), } } // WithLogFields returns a copy of parent with the supplied key-values added to the logger func WithLogFields(parent *Context, fields logrus.Fields) *Context { return &Context{ - Context: parent.Context, - Log: parent.Log.WithFields(fields), + Context: parent.Context, + FieldLogger: parent.FieldLogger.WithFields(fields), } } @@ -91,8 +91,8 @@ func WithLogFields(parent *Context, fields logrus.Fields) *Context { // val. It is analogous to context.WithValue() func WithValue(parent *Context, key, val any) *Context { return &Context{ - Context: context.WithValue(parent, key, val), - Log: parent.Log, + Context: context.WithValue(parent, key, val), + FieldLogger: parent.FieldLogger, } } @@ -101,7 +101,7 @@ func WithValue(parent *Context, key, val any) *Context { func ErrGroup(ctx *Context) (*errgroup.Group, *Context) { group, goctx := errgroup.WithContext(ctx) return group, &Context{ - Context: goctx, - Log: ctx.Log, + Context: goctx, + FieldLogger: ctx.FieldLogger, } } diff --git a/internal/common/armadacontext/armada_context_test.go b/internal/common/armadacontext/armada_context_test.go index a98d7b611df..4cda401c1b1 100644 --- a/internal/common/armadacontext/armada_context_test.go +++ b/internal/common/armadacontext/armada_context_test.go @@ -15,7 +15,7 @@ var defaultLogger = logrus.WithField("foo", "bar") func TestNew(t *testing.T) { ctx := New(context.Background(), defaultLogger) - require.Equal(t, defaultLogger, ctx.Log) + require.Equal(t, defaultLogger, ctx.FieldLogger) require.Equal(t, context.Background(), ctx.Context) } @@ -23,7 +23,7 @@ func TestFromGrpcContext(t *testing.T) { grpcCtx := ctxlogrus.ToContext(context.Background(), defaultLogger) ctx := FromGrpcCtx(grpcCtx) require.Equal(t, grpcCtx, ctx.Context) - require.Equal(t, defaultLogger, ctx.Log) + require.Equal(t, defaultLogger, ctx.FieldLogger) } func TestBackground(t *testing.T) { @@ -39,13 +39,13 @@ func TestTODO(t *testing.T) { func TestWithLogField(t *testing.T) { ctx := WithLogField(Background(), "fish", "chips") require.Equal(t, context.Background(), ctx.Context) - require.Equal(t, logrus.Fields{"fish": "chips"}, ctx.Log.Data) + require.Equal(t, logrus.Fields{"fish": "chips"}, ctx.FieldLogger.(*logrus.Entry).Data) } func TestWithLogFields(t *testing.T) { ctx := WithLogFields(Background(), logrus.Fields{"fish": "chips", "salt": "pepper"}) require.Equal(t, context.Background(), ctx.Context) - require.Equal(t, logrus.Fields{"fish": "chips", "salt": "pepper"}, ctx.Log.Data) + require.Equal(t, logrus.Fields{"fish": "chips", "salt": "pepper"}, ctx.FieldLogger.(*logrus.Entry).Data) } func TestWithTimeout(t *testing.T) { diff --git a/internal/common/logging/stacktrace.go b/internal/common/logging/stacktrace.go index cdcf4aef525..7d546915b31 100644 --- a/internal/common/logging/stacktrace.go +++ b/internal/common/logging/stacktrace.go @@ -10,9 +10,9 @@ type stackTracer interface { StackTrace() errors.StackTrace } -// WithStacktrace returns a new logrus.Entry obtained by adding error information and, if available, a stack trace -// as fields to the provided logrus.Entry. -func WithStacktrace(logger *logrus.Entry, err error) *logrus.Entry { +// WithStacktrace returns a new logrus.FieldLogger obtained by adding error information and, if available, a stack trace +// as fields to the provided logrus.FieldLogger. +func WithStacktrace(logger logrus.FieldLogger, err error) logrus.FieldLogger { logger = logger.WithError(err) if stackErr, ok := err.(stackTracer); ok { return logger.WithField("stacktrace", stackErr.StackTrace()) diff --git a/internal/scheduler/api.go b/internal/scheduler/api.go index a31eba85f5e..533abc4b728 100644 --- a/internal/scheduler/api.go +++ b/internal/scheduler/api.go @@ -103,7 +103,7 @@ func (srv *ExecutorApi) LeaseJobRuns(stream executorapi.ExecutorApi_LeaseJobRuns if err != nil { return err } - ctx.Log.Infof( + ctx.Infof( "executor currently has %d job runs; sending %d cancellations and %d new runs", len(requestRuns), len(runsToCancel), len(newRuns), ) @@ -226,7 +226,7 @@ func (srv *ExecutorApi) executorFromLeaseRequest(ctx *armadacontext.Context, req now := srv.clock.Now().UTC() for _, nodeInfo := range req.Nodes { if node, err := api.NewNodeFromNodeInfo(nodeInfo, req.ExecutorId, srv.allowedPriorities, now); err != nil { - logging.WithStacktrace(ctx.Log, err).Warnf( + logging.WithStacktrace(ctx, err).Warnf( "skipping node %s from executor %s", nodeInfo.GetName(), req.GetExecutorId(), ) } else { diff --git a/internal/scheduler/common.go b/internal/scheduler/common.go index 4b0bc6e2940..1fbf71b61fa 100644 --- a/internal/scheduler/common.go +++ b/internal/scheduler/common.go @@ -24,6 +24,9 @@ type SchedulerResult struct { PreemptedJobs []interfaces.LegacySchedulerJob // Queued jobs that should be scheduled. ScheduledJobs []interfaces.LegacySchedulerJob + // Queued jobs that could not be scheduled. + // This is used to fail jobs that could not schedule above `minimumGangCardinality`. + FailedJobs []interfaces.LegacySchedulerJob // For each preempted job, maps the job id to the id of the node on which the job was running. // For each scheduled job, maps the job id to the id of the node on which the job should be scheduled. NodeIdByJobId map[string]string @@ -32,9 +35,10 @@ type SchedulerResult struct { SchedulingContexts []*schedulercontext.SchedulingContext } -func NewSchedulerResult[S ~[]T, T interfaces.LegacySchedulerJob]( +func NewSchedulerResultForTest[S ~[]T, T interfaces.LegacySchedulerJob]( preemptedJobs S, scheduledJobs S, + failedJobs S, nodeIdByJobId map[string]string, ) *SchedulerResult { castPreemptedJobs := make([]interfaces.LegacySchedulerJob, len(preemptedJobs)) @@ -45,10 +49,15 @@ func NewSchedulerResult[S ~[]T, T interfaces.LegacySchedulerJob]( for i, job := range scheduledJobs { castScheduledJobs[i] = job } + castFailedJobs := make([]interfaces.LegacySchedulerJob, len(failedJobs)) + for i, job := range failedJobs { + castFailedJobs[i] = job + } return &SchedulerResult{ PreemptedJobs: castPreemptedJobs, ScheduledJobs: castScheduledJobs, NodeIdByJobId: nodeIdByJobId, + FailedJobs: castFailedJobs, } } @@ -72,6 +81,16 @@ func ScheduledJobsFromSchedulerResult[T interfaces.LegacySchedulerJob](sr *Sched return rv } +// FailedJobsFromScheduleResult returns the slice of scheduled jobs in the result, +// cast to type T. +func FailedJobsFromSchedulerResult[T interfaces.LegacySchedulerJob](sr *SchedulerResult) []T { + rv := make([]T, len(sr.FailedJobs)) + for i, job := range sr.FailedJobs { + rv[i] = job.(T) + } + return rv +} + // JobsSummary returns a string giving an overview of the provided jobs meant for logging. // For example: "affected queues [A, B]; resources {A: {cpu: 1}, B: {cpu: 2}}; jobs [jobAId, jobBId]". func JobsSummary(jobs []interfaces.LegacySchedulerJob) string { @@ -132,22 +151,22 @@ func GangIdAndCardinalityFromLegacySchedulerJob(job interfaces.LegacySchedulerJo // GangIdAndCardinalityFromAnnotations returns a tuple (gangId, gangCardinality, gangMinimumCardinality, isGangJob, error). func GangIdAndCardinalityFromAnnotations(annotations map[string]string) (string, int, int, bool, error) { if annotations == nil { - return "", 0, 0, false, nil + return "", 1, 1, false, nil } gangId, ok := annotations[configuration.GangIdAnnotation] if !ok { - return "", 0, 0, false, nil + return "", 1, 1, false, nil } gangCardinalityString, ok := annotations[configuration.GangCardinalityAnnotation] if !ok { - return "", 0, 0, false, errors.Errorf("missing annotation %s", configuration.GangCardinalityAnnotation) + return "", 1, 1, false, errors.Errorf("missing annotation %s", configuration.GangCardinalityAnnotation) } gangCardinality, err := strconv.Atoi(gangCardinalityString) if err != nil { - return "", 0, 0, false, errors.WithStack(err) + return "", 1, 1, false, errors.WithStack(err) } if gangCardinality <= 0 { - return "", 0, 0, false, errors.Errorf("gang cardinality is non-positive %d", gangCardinality) + return "", 1, 1, false, errors.Errorf("gang cardinality is non-positive %d", gangCardinality) } gangMinimumCardinalityString, ok := annotations[configuration.GangMinimumCardinalityAnnotation] if !ok { @@ -156,10 +175,10 @@ func GangIdAndCardinalityFromAnnotations(annotations map[string]string) (string, } else { gangMinimumCardinality, err := strconv.Atoi(gangMinimumCardinalityString) if err != nil { - return "", 0, 0, false, errors.WithStack(err) + return "", 1, 1, false, errors.WithStack(err) } if gangMinimumCardinality <= 0 { - return "", 0, 0, false, errors.Errorf("gang minimum cardinality is non-positive %d", gangMinimumCardinality) + return "", 1, 1, false, errors.Errorf("gang minimum cardinality is non-positive %d", gangMinimumCardinality) } return gangId, gangCardinality, gangMinimumCardinality, true, nil } diff --git a/internal/scheduler/context/context.go b/internal/scheduler/context/context.go index 29c12bbc374..8a52b497735 100644 --- a/internal/scheduler/context/context.go +++ b/internal/scheduler/context/context.go @@ -226,15 +226,20 @@ func (sctx *SchedulingContext) ReportString(verbosity int32) string { func (sctx *SchedulingContext) AddGangSchedulingContext(gctx *GangSchedulingContext) (bool, error) { allJobsEvictedInThisRound := true allJobsSuccessful := true + numberOfSuccessfulJobs := 0 for _, jctx := range gctx.JobSchedulingContexts { evictedInThisRound, err := sctx.AddJobSchedulingContext(jctx) if err != nil { return false, err } allJobsEvictedInThisRound = allJobsEvictedInThisRound && evictedInThisRound - allJobsSuccessful = allJobsSuccessful && jctx.IsSuccessful() + isSuccess := jctx.IsSuccessful() + allJobsSuccessful = allJobsSuccessful && isSuccess + if isSuccess { + numberOfSuccessfulJobs++ + } } - if allJobsSuccessful && !allJobsEvictedInThisRound { + if numberOfSuccessfulJobs >= gctx.GangMinCardinality && !allJobsEvictedInThisRound { sctx.NumScheduledGangs++ } return allJobsEvictedInThisRound, nil @@ -458,15 +463,6 @@ func (qctx *QueueSchedulingContext) ReportString(verbosity int32) string { return sb.String() } -func (qctx *QueueSchedulingContext) AddGangSchedulingContext(gctx *GangSchedulingContext) error { - for _, jctx := range gctx.JobSchedulingContexts { - if _, err := qctx.AddJobSchedulingContext(jctx); err != nil { - return err - } - } - return nil -} - // AddJobSchedulingContext adds a job scheduling context. // Automatically updates scheduled resources. func (qctx *QueueSchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContext) (bool, error) { @@ -542,6 +538,7 @@ type GangSchedulingContext struct { TotalResourceRequests schedulerobjects.ResourceList AllJobsEvicted bool NodeUniformityLabel string + GangMinCardinality int } func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingContext { @@ -550,12 +547,14 @@ func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingCont queue := "" priorityClassName := "" nodeUniformityLabel := "" + gangMinCardinality := 1 if len(jctxs) > 0 { queue = jctxs[0].Job.GetQueue() priorityClassName = jctxs[0].Job.GetPriorityClassName() if jctxs[0].PodRequirements != nil { nodeUniformityLabel = jctxs[0].PodRequirements.Annotations[configuration.GangNodeUniformityLabelAnnotation] } + gangMinCardinality = jctxs[0].GangMinCardinality } allJobsEvicted := true totalResourceRequests := schedulerobjects.NewResourceList(4) @@ -571,6 +570,7 @@ func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingCont TotalResourceRequests: totalResourceRequests, AllJobsEvicted: allJobsEvicted, NodeUniformityLabel: nodeUniformityLabel, + GangMinCardinality: gangMinCardinality, } } @@ -600,6 +600,10 @@ type JobSchedulingContext struct { UnschedulableReason string // Pod scheduling contexts for the individual pods that make up the job. PodSchedulingContext *PodSchedulingContext + // The minimum size of the gang associated with this job. + GangMinCardinality int + // If set, indicates this job should be failed back to the client when the gang is scheduled. + ShouldFail bool } func (jctx *JobSchedulingContext) String() string { @@ -615,6 +619,7 @@ func (jctx *JobSchedulingContext) String() string { if jctx.PodSchedulingContext != nil { fmt.Fprint(w, jctx.PodSchedulingContext.String()) } + fmt.Fprintf(w, "GangMinCardinality:\t%d\n", jctx.GangMinCardinality) w.Flush() return sb.String() } @@ -623,15 +628,25 @@ func (jctx *JobSchedulingContext) IsSuccessful() bool { return jctx.UnschedulableReason == "" } -func JobSchedulingContextsFromJobs[J interfaces.LegacySchedulerJob](priorityClasses map[string]types.PriorityClass, jobs []J) []*JobSchedulingContext { +func JobSchedulingContextsFromJobs[J interfaces.LegacySchedulerJob](priorityClasses map[string]types.PriorityClass, jobs []J, extractGangInfo func(map[string]string) (string, int, int, bool, error)) []*JobSchedulingContext { jctxs := make([]*JobSchedulingContext, len(jobs)) timestamp := time.Now() + for i, job := range jobs { + // TODO: Move min cardinality to gang context only and remove from here. + // Requires re-phrasing nodedb in terms of gang context, as well as feeding the value extracted from the annotations downstream. + _, _, gangMinCardinality, _, err := extractGangInfo(job.GetAnnotations()) + if err != nil { + gangMinCardinality = 1 + } + jctxs[i] = &JobSchedulingContext{ - Created: timestamp, - JobId: job.GetId(), - Job: job, - PodRequirements: job.GetPodRequirements(priorityClasses), + Created: timestamp, + JobId: job.GetId(), + Job: job, + PodRequirements: job.GetPodRequirements(priorityClasses), + GangMinCardinality: gangMinCardinality, + ShouldFail: false, } } return jctxs diff --git a/internal/scheduler/context/context_test.go b/internal/scheduler/context/context_test.go index 67fdabd483f..6fe905bfeaf 100644 --- a/internal/scheduler/context/context_test.go +++ b/internal/scheduler/context/context_test.go @@ -89,8 +89,9 @@ func testNSmallCpuJobSchedulingContext(queue, priorityClassName string, n int) [ func testSmallCpuJobSchedulingContext(queue, priorityClassName string) *JobSchedulingContext { job := testfixtures.Test1Cpu4GiJob(queue, priorityClassName) return &JobSchedulingContext{ - JobId: job.GetId(), - Job: job, - PodRequirements: job.GetPodRequirements(testfixtures.TestPriorityClasses), + JobId: job.GetId(), + Job: job, + PodRequirements: job.GetPodRequirements(testfixtures.TestPriorityClasses), + GangMinCardinality: 1, } } diff --git a/internal/scheduler/database/db_pruner.go b/internal/scheduler/database/db_pruner.go index 9ea8075a40d..8da7dd7935d 100644 --- a/internal/scheduler/database/db_pruner.go +++ b/internal/scheduler/database/db_pruner.go @@ -1,13 +1,13 @@ package database import ( - ctx "context" "time" "github.com/jackc/pgx/v5" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/clock" + + "github.com/armadaproject/armada/internal/common/armadacontext" ) // PruneDb removes completed jobs (and related runs and errors) from the database if their `lastUpdateTime` @@ -15,7 +15,7 @@ import ( // Jobs are deleted in batches across transactions. This means that if this job fails midway through, it still // may have deleted some jobs. // The function will run until the supplied context is cancelled. -func PruneDb(ctx ctx.Context, db *pgx.Conn, batchLimit int, keepAfterCompletion time.Duration, clock clock.Clock) error { +func PruneDb(ctx *armadacontext.Context, db *pgx.Conn, batchLimit int, keepAfterCompletion time.Duration, clock clock.Clock) error { start := time.Now() cutOffTime := clock.Now().Add(-keepAfterCompletion) @@ -40,11 +40,11 @@ func PruneDb(ctx ctx.Context, db *pgx.Conn, batchLimit int, keepAfterCompletion return errors.WithStack(err) } if totalJobsToDelete == 0 { - log.Infof("Found no jobs to be deleted. Exiting") + ctx.Infof("Found no jobs to be deleted. Exiting") return nil } - log.Infof("Found %d jobs to be deleted", totalJobsToDelete) + ctx.Infof("Found %d jobs to be deleted", totalJobsToDelete) // create temp table to hold a batch of results _, err = db.Exec(ctx, "CREATE TEMP TABLE batch (job_id TEXT);") @@ -93,9 +93,10 @@ func PruneDb(ctx ctx.Context, db *pgx.Conn, batchLimit int, keepAfterCompletion taken := time.Now().Sub(batchStart) jobsDeleted += batchSize - log.Infof("Deleted %d jobs in %s. Deleted %d jobs out of %d", batchSize, taken, jobsDeleted, totalJobsToDelete) + ctx. + Infof("Deleted %d jobs in %s. Deleted %d jobs out of %d", batchSize, taken, jobsDeleted, totalJobsToDelete) } taken := time.Now().Sub(start) - log.Infof("Deleted %d jobs in %s", jobsDeleted, taken) + ctx.Infof("Deleted %d jobs in %s", jobsDeleted, taken) return nil } diff --git a/internal/scheduler/database/util.go b/internal/scheduler/database/util.go index 618c32c8efb..af338ee3b42 100644 --- a/internal/scheduler/database/util.go +++ b/internal/scheduler/database/util.go @@ -6,7 +6,6 @@ import ( "time" "github.com/jackc/pgx/v5/pgxpool" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/database" @@ -25,7 +24,7 @@ func Migrate(ctx *armadacontext.Context, db database.Querier) error { if err != nil { return err } - log.Infof("Updated scheduler database in %s", time.Now().Sub(start)) + ctx.Infof("Updated scheduler database in %s", time.Now().Sub(start)) return nil } diff --git a/internal/scheduler/gang_scheduler.go b/internal/scheduler/gang_scheduler.go index fb9a3add118..f81e6bcaff4 100644 --- a/internal/scheduler/gang_scheduler.go +++ b/internal/scheduler/gang_scheduler.go @@ -38,11 +38,55 @@ func (sch *GangScheduler) SkipUnsuccessfulSchedulingKeyCheck() { sch.skipUnsuccessfulSchedulingKeyCheck = true } -func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { - // Exit immediately if this is a new gang and we've exceeded any round limits. +func (sch *GangScheduler) updateGangSchedulingContextOnFailure(gctx *schedulercontext.GangSchedulingContext, gangAddedToSchedulingContext bool, unschedulableReason string) (err error) { + if gangAddedToSchedulingContext { + failedJobs := util.Map(gctx.JobSchedulingContexts, func(jctx *schedulercontext.JobSchedulingContext) interfaces.LegacySchedulerJob { return jctx.Job }) + if _, err = sch.schedulingContext.EvictGang(failedJobs); err != nil { + return + } + } + + for _, jctx := range gctx.JobSchedulingContexts { + jctx.UnschedulableReason = unschedulableReason + } + + if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { + return + } + + // Register unfeasible scheduling keys. // - // Because this check occurs before adding the gctx to the sctx, - // the round limits can be exceeded by one gang. + // Only record unfeasible scheduling keys for single-job gangs. + // Since a gang may be unschedulable even if all its members are individually schedulable. + if !sch.skipUnsuccessfulSchedulingKeyCheck && gctx.Cardinality() == 1 { + jctx := gctx.JobSchedulingContexts[0] + schedulingKey := sch.schedulingContext.SchedulingKeyFromLegacySchedulerJob(jctx.Job) + if _, ok := sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey]; !ok { + // Keep the first jctx for each unfeasible schedulingKey. + sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey] = jctx + } + } + + return +} + +func (sch *GangScheduler) updateGangSchedulingContextOnSuccess(gctx *schedulercontext.GangSchedulingContext, gangAddedToSchedulingContext bool) (err error) { + if gangAddedToSchedulingContext { + jobs := util.Map(gctx.JobSchedulingContexts, func(jctx *schedulercontext.JobSchedulingContext) interfaces.LegacySchedulerJob { return jctx.Job }) + if _, err = sch.schedulingContext.EvictGang(jobs); err != nil { + return + } + } + + if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { + return + } + + return +} + +func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { + // Exit immediately if this is a new gang and we've hit any round limits. if !gctx.AllJobsEvicted { if ok, unschedulableReason, err = sch.constraints.CheckRoundConstraints(sch.schedulingContext, gctx.Queue); err != nil || !ok { return @@ -66,36 +110,15 @@ func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulerco } } - // Process unschedulable jobs. - if !ok { - // Register the job as unschedulable. If the job was added to the context, remove it first. - if gangAddedToSchedulingContext { - jobs := util.Map(gctx.JobSchedulingContexts, func(jctx *schedulercontext.JobSchedulingContext) interfaces.LegacySchedulerJob { return jctx.Job }) - if _, err = sch.schedulingContext.EvictGang(jobs); err != nil { - return - } - } - for _, jctx := range gctx.JobSchedulingContexts { - jctx.UnschedulableReason = unschedulableReason - } - if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { - return - } - - // Register unfeasible scheduling keys. - // - // Only record unfeasible scheduling keys for single-job gangs. - // Since a gang may be unschedulable even if all its members are individually schedulable. - if !sch.skipUnsuccessfulSchedulingKeyCheck && gctx.Cardinality() == 1 { - jctx := gctx.JobSchedulingContexts[0] - schedulingKey := sch.schedulingContext.SchedulingKeyFromLegacySchedulerJob(jctx.Job) - if _, ok := sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey]; !ok { - // Keep the first jctx for each unfeasible schedulingKey. - sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey] = jctx - } - } + if ok { + err = sch.updateGangSchedulingContextOnSuccess(gctx, gangAddedToSchedulingContext) + } else { + err = sch.updateGangSchedulingContextOnFailure(gctx, gangAddedToSchedulingContext, unschedulableReason) } + + return }() + if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { return } @@ -186,23 +209,38 @@ func (sch *GangScheduler) tryScheduleGang(ctx *armadacontext.Context, gctx *sche return } -func (sch *GangScheduler) tryScheduleGangWithTxn(ctx *armadacontext.Context, txn *memdb.Txn, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { - if ok, err = sch.nodeDb.ScheduleManyWithTxn(txn, gctx.JobSchedulingContexts); err != nil { - return - } else if !ok { - for _, jctx := range gctx.JobSchedulingContexts { - if jctx.PodSchedulingContext != nil { - // Clear any node bindings on failure to schedule. - jctx.PodSchedulingContext.NodeId = "" +func clearNodeBindings(jctx *schedulercontext.JobSchedulingContext) { + if jctx.PodSchedulingContext != nil { + // Clear any node bindings on failure to schedule. + jctx.PodSchedulingContext.NodeId = "" + } +} + +func (sch *GangScheduler) tryScheduleGangWithTxn(_ *armadacontext.Context, txn *memdb.Txn, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { + if ok, err = sch.nodeDb.ScheduleManyWithTxn(txn, gctx.JobSchedulingContexts); err == nil { + if !ok { + for _, jctx := range gctx.JobSchedulingContexts { + clearNodeBindings(jctx) + } + + if gctx.Cardinality() > 1 { + unschedulableReason = "unable to schedule gang since minimum cardinality not met" + } else { + unschedulableReason = "job does not fit on any node" } - } - if gctx.Cardinality() > 1 { - unschedulableReason = "at least one job in the gang does not fit on any node" } else { - unschedulableReason = "job does not fit on any node" + // When a gang schedules successfully, update state for failed jobs if they exist. + for _, jctx := range gctx.JobSchedulingContexts { + if jctx.ShouldFail { + clearNodeBindings(jctx) + jctx.UnschedulableReason = "job does not fit on any node" + } + } } + return } + return } diff --git a/internal/scheduler/gang_scheduler_test.go b/internal/scheduler/gang_scheduler_test.go index cc79703d2b2..e0b89dafbeb 100644 --- a/internal/scheduler/gang_scheduler_test.go +++ b/internal/scheduler/gang_scheduler_test.go @@ -34,39 +34,74 @@ func TestGangScheduler(t *testing.T) { Gangs [][]*jobdb.Job // Indices of gangs expected to be scheduled. ExpectedScheduledIndices []int + // Cumulative number of jobs we expect to schedule successfully. + // Each index `i` is the expected value when processing gang `i`. + ExpectedScheduledJobs []int }{ "simple success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32}, }, "simple failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, + }, + "simple success where min cardinality is met": { + SchedulingConfig: testfixtures.TestSchedulingConfig(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + testfixtures.WithGangAnnotationsJobsAndMinCardinality(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 40), 32), + }, + ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32}, + }, + "simple failure where min cardinality is not met": { + SchedulingConfig: testfixtures.TestSchedulingConfig(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + testfixtures.WithGangAnnotationsJobsAndMinCardinality(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 40), 33), + }, + ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "one success and one failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32, 32}, + }, + "one success and one failure using min cardinality": { + SchedulingConfig: testfixtures.TestSchedulingConfig(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + testfixtures.WithGangAnnotationsJobsAndMinCardinality(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), 32), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + }, + ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32, 32}, }, "multiple nodes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{64}, }, "MaximumResourceFractionToSchedule": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( @@ -75,11 +110,12 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 16), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 16)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8)), }, ExpectedScheduledIndices: []int{0, 1}, + ExpectedScheduledJobs: []int{8, 24, 24}, }, "MaximumResourceFractionToScheduleByPool": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( @@ -91,13 +127,14 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: []int{0, 1, 2}, + ExpectedScheduledJobs: []int{1, 2, 3, 3, 3}, }, "MaximumResourceFractionToScheduleByPool non-existing pool": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( @@ -109,13 +146,14 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: []int{0, 1, 2, 3}, + ExpectedScheduledJobs: []int{1, 2, 3, 4, 4}, }, "MaximumResourceFractionPerQueue": { SchedulingConfig: testfixtures.WithPerPriorityLimitsConfig( @@ -129,16 +167,17 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 2), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 3), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 3), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 4), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 4), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 5), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 2)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 3)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 3)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 4)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 4)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 5)), }, ExpectedScheduledIndices: []int{0, 2, 4, 6}, + ExpectedScheduledJobs: []int{1, 1, 3, 3, 6, 6, 10, 10}, }, "resolution has no impact on jobs of size a multiple of the resolution": { SchedulingConfig: testfixtures.WithIndexedResourcesConfig( @@ -150,14 +189,15 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 5), + ExpectedScheduledJobs: testfixtures.IntRange(1, 6), }, "jobs of size not a multiple of the resolution blocks scheduling new jobs": { SchedulingConfig: testfixtures.WithIndexedResourcesConfig( @@ -169,12 +209,13 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 2), + ExpectedScheduledJobs: []int{1, 2, 3, 3}, }, "consider all nodes in the bucket": { SchedulingConfig: testfixtures.WithIndexedResourcesConfig( @@ -208,9 +249,10 @@ func TestGangScheduler(t *testing.T) { ), ), Gangs: [][]*jobdb.Job{ - testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{1}, }, "NodeUniformityLabel set but not indexed": { SchedulingConfig: testfixtures.TestSchedulingConfig(), @@ -219,12 +261,14 @@ func TestGangScheduler(t *testing.T) { testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - ), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs( + "foo", + testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + )), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "NodeUniformityLabel not set": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig( @@ -233,12 +277,14 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - ), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs( + "foo", + testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + )), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "NodeUniformityLabel insufficient capacity": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig( @@ -256,12 +302,12 @@ func TestGangScheduler(t *testing.T) { ), ), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs("foo", testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3)), ), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "NodeUniformityLabel": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig( @@ -291,12 +337,14 @@ func TestGangScheduler(t *testing.T) { ), ), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 4), - ), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs( + "foo", + testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 4), + )), }, ExpectedScheduledIndices: []int{0}, + ExpectedScheduledJobs: []int{4}, }, } for name, tc := range tests { @@ -369,8 +417,9 @@ func TestGangScheduler(t *testing.T) { require.NoError(t, err) var actualScheduledIndices []int + scheduledGangs := 0 for i, gang := range tc.Gangs { - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, gang) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, gang, GangIdAndCardinalityFromAnnotations) gctx := schedulercontext.NewGangSchedulingContext(jctxs) ok, reason, err := sch.Schedule(armadacontext.Background(), gctx) require.NoError(t, err) @@ -394,8 +443,36 @@ func TestGangScheduler(t *testing.T) { "node uniformity constraint not met: %s", nodeUniformityLabelValues, ) } + + // Verify any excess jobs that failed have the correct state set + for _, jctx := range jctxs { + if jctx.ShouldFail { + if jctx.PodSchedulingContext != nil { + require.Equal(t, "", jctx.PodSchedulingContext.NodeId) + } + require.Equal(t, "job does not fit on any node", jctx.UnschedulableReason) + } + } + + // Verify accounting + scheduledGangs++ + require.Equal(t, scheduledGangs, sch.schedulingContext.NumScheduledGangs) + require.Equal(t, tc.ExpectedScheduledJobs[i], sch.schedulingContext.NumScheduledJobs) + require.Equal(t, 0, sch.schedulingContext.NumEvictedJobs) } else { require.NotEmpty(t, reason) + + // Verify all jobs have been correctly unbound from nodes + for _, jctx := range jctxs { + if jctx.PodSchedulingContext != nil { + require.Equal(t, "", jctx.PodSchedulingContext.NodeId) + } + } + + // Verify accounting + require.Equal(t, scheduledGangs, sch.schedulingContext.NumScheduledGangs) + require.Equal(t, tc.ExpectedScheduledJobs[i], sch.schedulingContext.NumScheduledJobs) + require.Equal(t, 0, sch.schedulingContext.NumEvictedJobs) } } assert.Equal(t, tc.ExpectedScheduledIndices, actualScheduledIndices) diff --git a/internal/scheduler/leader.go b/internal/scheduler/leader.go index 0482184a7a8..714cf243f52 100644 --- a/internal/scheduler/leader.go +++ b/internal/scheduler/leader.go @@ -145,7 +145,7 @@ func (lc *KubernetesLeaderController) Run(ctx *armadacontext.Context) error { return ctx.Err() default: lock := lc.getNewLock() - ctx.Log.Infof("attempting to become leader") + ctx.Infof("attempting to become leader") leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ Lock: lock, ReleaseOnCancel: true, @@ -154,14 +154,14 @@ func (lc *KubernetesLeaderController) Run(ctx *armadacontext.Context) error { RetryPeriod: lc.config.RetryPeriod, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: func(c context.Context) { - ctx.Log.Infof("I am now leader") + ctx.Infof("I am now leader") lc.token.Store(NewLeaderToken()) for _, listener := range lc.listeners { listener.onStartedLeading(ctx) } }, OnStoppedLeading: func() { - ctx.Log.Infof("I am no longer leader") + ctx.Infof("I am no longer leader") lc.token.Store(InvalidLeaderToken()) for _, listener := range lc.listeners { listener.onStoppedLeading() @@ -174,7 +174,7 @@ func (lc *KubernetesLeaderController) Run(ctx *armadacontext.Context) error { }, }, }) - ctx.Log.Infof("leader election round finished") + ctx.Infof("leader election round finished") } } } diff --git a/internal/scheduler/metrics.go b/internal/scheduler/metrics.go index a7fb2f08c78..168295ff91f 100644 --- a/internal/scheduler/metrics.go +++ b/internal/scheduler/metrics.go @@ -7,10 +7,10 @@ import ( "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/clock" "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/internal/common/logging" commonmetrics "github.com/armadaproject/armada/internal/common/metrics" "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/scheduler/database" @@ -78,16 +78,18 @@ func NewMetricsCollector( // Run enters s a loop which updates the metrics every refreshPeriod until the supplied context is cancelled func (c *MetricsCollector) Run(ctx *armadacontext.Context) error { ticker := c.clock.NewTicker(c.refreshPeriod) - log.Infof("Will update metrics every %s", c.refreshPeriod) + ctx.Infof("Will update metrics every %s", c.refreshPeriod) for { select { case <-ctx.Done(): - log.Debugf("Context cancelled, returning..") + ctx.Debugf("Context cancelled, returning..") return nil case <-ticker.C(): err := c.refresh(ctx) if err != nil { - log.WithError(err).Warnf("error refreshing metrics state") + logging. + WithStacktrace(ctx, err). + Warnf("error refreshing metrics state") } } } @@ -109,7 +111,7 @@ func (c *MetricsCollector) Collect(metrics chan<- prometheus.Metric) { } func (c *MetricsCollector) refresh(ctx *armadacontext.Context) error { - log.Debugf("Refreshing prometheus metrics") + ctx.Debugf("Refreshing prometheus metrics") start := time.Now() queueMetrics, err := c.updateQueueMetrics(ctx) if err != nil { @@ -121,7 +123,7 @@ func (c *MetricsCollector) refresh(ctx *armadacontext.Context) error { } allMetrics := append(queueMetrics, clusterMetrics...) c.state.Store(allMetrics) - log.Debugf("Refreshed prometheus metrics in %s", time.Since(start)) + ctx.Debugf("Refreshed prometheus metrics in %s", time.Since(start)) return nil } @@ -154,7 +156,7 @@ func (c *MetricsCollector) updateQueueMetrics(ctx *armadacontext.Context) ([]pro } qs, ok := provider.queueStates[job.Queue()] if !ok { - log.Warnf("job %s is in queue %s, but this queue does not exist; skipping", job.Id(), job.Queue()) + ctx.Warnf("job %s is in queue %s, but this queue does not exist; skipping", job.Id(), job.Queue()) continue } @@ -181,7 +183,7 @@ func (c *MetricsCollector) updateQueueMetrics(ctx *armadacontext.Context) ([]pro timeInState = currentTime.Sub(time.Unix(0, run.Created())) recorder = qs.runningJobRecorder } else { - log.Warnf("Job %s is marked as leased but has no runs", job.Id()) + ctx.Warnf("Job %s is marked as leased but has no runs", job.Id()) } recorder.RecordJobRuntime(pool, priorityClass, timeInState) recorder.RecordResources(pool, priorityClass, jobResources) diff --git a/internal/scheduler/nodedb/nodedb.go b/internal/scheduler/nodedb/nodedb.go index a2351d1e75f..e529d870ff6 100644 --- a/internal/scheduler/nodedb/nodedb.go +++ b/internal/scheduler/nodedb/nodedb.go @@ -488,13 +488,9 @@ func NodeJobDiff(txnA, txnB *memdb.Txn) (map[string]*Node, map[string]*Node, err return preempted, scheduled, nil } -// ScheduleMany assigns a set of jobs to nodes. The assignment is atomic, i.e., either all jobs are -// successfully assigned to nodes or none are. The returned bool indicates whether assignment -// succeeded (true) or not (false). -// -// This method sets the PodSchedulingContext field on each JobSchedulingContext that it attempts to -// schedule; if it returns early (e.g., because it finds an unschedulable JobSchedulingContext), -// then this field will not be set on the remaining items. +// ScheduleMany assigns a set of jobs to nodes. +// If N jobs can be scheduled, where N >= `GangMinCardinality`, it will return true, nil and set ShouldFail on any excess jobs. +// Otherwise, it will return false, nil. // TODO: Pass through contexts to support timeouts. func (nodeDb *NodeDb) ScheduleMany(jctxs []*schedulercontext.JobSchedulingContext) (bool, error) { txn := nodeDb.db.Txn(true) @@ -507,25 +503,42 @@ func (nodeDb *NodeDb) ScheduleMany(jctxs []*schedulercontext.JobSchedulingContex return ok, err } +// TODO: Remove me once we re-phrase nodedb in terms of gang context (and therefore can just take this value from the gang scheduling context provided) +func gangMinCardinality(jctxs []*schedulercontext.JobSchedulingContext) int { + if len(jctxs) > 0 { + return jctxs[0].GangMinCardinality + } else { + return 1 + } +} + func (nodeDb *NodeDb) ScheduleManyWithTxn(txn *memdb.Txn, jctxs []*schedulercontext.JobSchedulingContext) (bool, error) { // Attempt to schedule pods one by one in a transaction. + cumulativeScheduled := 0 + gangMinCardinality := gangMinCardinality(jctxs) + for _, jctx := range jctxs { + // Defensively reset `ShouldFail` (this should always be false as the state is re-constructed per cycle but just in case) + jctx.ShouldFail = false + node, err := nodeDb.SelectNodeForJobWithTxn(txn, jctx) if err != nil { return false, err } + if node == nil { + // Indicates that when the min cardinality is met, we should fail this job back to the client. + jctx.ShouldFail = true + continue + } + // If we found a node for this pod, bind it and continue to the next pod. - if node != nil { - if node, err := bindJobToNode(nodeDb.priorityClasses, jctx.Job, node); err != nil { + if node, err := bindJobToNode(nodeDb.priorityClasses, jctx.Job, node); err != nil { + return false, err + } else { + if err := nodeDb.UpsertWithTxn(txn, node); err != nil { return false, err - } else { - if err := nodeDb.UpsertWithTxn(txn, node); err != nil { - return false, err - } } - } else { - return false, nil } // Once a job is scheduled, it should no longer be considered for preemption. @@ -534,7 +547,14 @@ func (nodeDb *NodeDb) ScheduleManyWithTxn(txn *memdb.Txn, jctxs []*schedulercont return false, err } } + + cumulativeScheduled++ } + + if cumulativeScheduled < gangMinCardinality { + return false, nil + } + return true, nil } diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 50f7f9c5a9c..8cc33f8188d 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -2,14 +2,17 @@ package nodedb import ( "fmt" + "strconv" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "github.com/armadaproject/armada/internal/armada/configuration" armadamaps "github.com/armadaproject/armada/internal/common/maps" schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" @@ -71,7 +74,7 @@ func TestSelectNodeForPod_NodeIdLabel_Success(t *testing.T) { map[string]string{schedulerconfig.NodeIdLabel: nodeId}, testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), ) - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) for _, jctx := range jctxs { txn := db.Txn(false) node, err := db.SelectNodeForJobWithTxn(txn, jctx) @@ -100,7 +103,7 @@ func TestSelectNodeForPod_NodeIdLabel_Failure(t *testing.T) { map[string]string{schedulerconfig.NodeIdLabel: "this node does not exist"}, testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), ) - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) for _, jctx := range jctxs { txn := db.Txn(false) node, err := db.SelectNodeForJobWithTxn(txn, jctx) @@ -435,7 +438,7 @@ func TestScheduleIndividually(t *testing.T) { nodeDb, err := newNodeDbWithNodes(tc.Nodes) require.NoError(t, err) - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, tc.Jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, tc.Jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) for i, jctx := range jctxs { ok, err := nodeDb.ScheduleMany([]*schedulercontext.JobSchedulingContext{jctx}) @@ -474,6 +477,9 @@ func TestScheduleIndividually(t *testing.T) { } func TestScheduleMany(t *testing.T) { + gangSuccess := testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)) + gangFailure := testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)) + tests := map[string]struct { // Nodes to schedule across. Nodes []*schedulerobjects.Node @@ -483,22 +489,30 @@ func TestScheduleMany(t *testing.T) { // For each group, whether we expect scheduling to succeed. ExpectSuccess []bool }{ + // Attempts to schedule 32 jobs with a minimum gang cardinality of 1 job. All jobs get scheduled. "simple success": { Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), - Jobs: [][]*jobdb.Job{testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)}, + Jobs: [][]*jobdb.Job{gangSuccess}, ExpectSuccess: []bool{true}, }, - "simple failure": { + // Attempts to schedule 33 jobs with a minimum gang cardinality of 32 jobs. One fails, but the overall result is a success. + "simple success with min cardinality": { Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), - Jobs: [][]*jobdb.Job{testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)}, + Jobs: [][]*jobdb.Job{testfixtures.WithGangAnnotationsJobsAndMinCardinality(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), 32)}, + ExpectSuccess: []bool{true}, + }, + // Attempts to schedule 33 jobs with a minimum gang cardinality of 33. The overall result fails. + "simple failure with min cardinality": { + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Jobs: [][]*jobdb.Job{gangFailure}, ExpectSuccess: []bool{false}, }, "correct rollback": { Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), + gangSuccess, + gangFailure, + gangSuccess, }, ExpectSuccess: []bool{true, false, true}, }, @@ -519,14 +533,27 @@ func TestScheduleMany(t *testing.T) { nodeDb, err := newNodeDbWithNodes(tc.Nodes) require.NoError(t, err) for i, jobs := range tc.Jobs { - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) - ok, err := nodeDb.ScheduleMany(jctxs) + minCardinalityStr, ok := jobs[0].GetAnnotations()[configuration.GangMinimumCardinalityAnnotation] + if !ok { + minCardinalityStr = "1" + } + minCardinality, err := strconv.Atoi(minCardinalityStr) + if err != nil { + minCardinality = 1 + } + extractGangInfo := func(_ map[string]string) (string, int, int, bool, error) { + id, _ := uuid.NewUUID() + return id.String(), 1, minCardinality, true, nil + } + + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, extractGangInfo) + ok, err = nodeDb.ScheduleMany(jctxs) require.NoError(t, err) assert.Equal(t, tc.ExpectSuccess[i], ok) for _, jctx := range jctxs { pctx := jctx.PodSchedulingContext require.NotNil(t, pctx) - if tc.ExpectSuccess[i] { + if tc.ExpectSuccess[i] && !jctx.ShouldFail { assert.NotEqual(t, "", pctx.NodeId) } } @@ -591,7 +618,7 @@ func benchmarkScheduleMany(b *testing.B, nodes []*schedulerobjects.Node, jobs [] b.ResetTimer() for n := 0; n < b.N; n++ { - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) txn := nodeDb.Txn(true) _, err := nodeDb.ScheduleManyWithTxn(txn, jctxs) txn.Abort() diff --git a/internal/scheduler/pool_assigner.go b/internal/scheduler/pool_assigner.go index 9ff1f9b140c..9d636570a61 100644 --- a/internal/scheduler/pool_assigner.go +++ b/internal/scheduler/pool_assigner.go @@ -135,10 +135,11 @@ func (p *DefaultPoolAssigner) AssignPool(j *jobdb.Job) (string, error) { nodeDb := e.nodeDb txn := nodeDb.Txn(true) jctx := &schedulercontext.JobSchedulingContext{ - Created: time.Now(), - JobId: j.GetId(), - Job: j, - PodRequirements: j.GetPodRequirements(p.priorityClasses), + Created: time.Now(), + JobId: j.GetId(), + Job: j, + PodRequirements: j.GetPodRequirements(p.priorityClasses), + GangMinCardinality: 1, } node, err := nodeDb.SelectNodeForJobWithTxn(txn, jctx) txn.Abort() diff --git a/internal/scheduler/preempting_queue_scheduler.go b/internal/scheduler/preempting_queue_scheduler.go index fd0c0d9e079..7be10464132 100644 --- a/internal/scheduler/preempting_queue_scheduler.go +++ b/internal/scheduler/preempting_queue_scheduler.go @@ -129,11 +129,11 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche sch.nodeEvictionProbability, func(ctx *armadacontext.Context, job interfaces.LegacySchedulerJob) bool { if job.GetAnnotations() == nil { - ctx.Log.Errorf("can't evict job %s: annotations not initialised", job.GetId()) + ctx.Errorf("can't evict job %s: annotations not initialised", job.GetId()) return false } if job.GetNodeSelector() == nil { - ctx.Log.Errorf("can't evict job %s: nodeSelector not initialised", job.GetId()) + ctx.Errorf("can't evict job %s: nodeSelector not initialised", job.GetId()) return false } if qctx, ok := sch.schedulingContext.QueueSchedulingContexts[job.GetQueue()]; ok { @@ -241,10 +241,10 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche return nil, err } if s := JobsSummary(preemptedJobs); s != "" { - ctx.Log.Infof("preempting running jobs; %s", s) + ctx.Infof("preempting running jobs; %s", s) } if s := JobsSummary(scheduledJobs); s != "" { - ctx.Log.Infof("scheduling new jobs; %s", s) + ctx.Infof("scheduling new jobs; %s", s) } if sch.enableAssertions { err := sch.assertions( @@ -260,6 +260,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche return &SchedulerResult{ PreemptedJobs: preemptedJobs, ScheduledJobs: scheduledJobs, + FailedJobs: schedulerResult.FailedJobs, NodeIdByJobId: sch.nodeIdByJobId, SchedulingContexts: []*schedulercontext.SchedulingContext{sch.schedulingContext}, }, nil @@ -805,7 +806,7 @@ func NewOversubscribedEvictor( }, jobFilter: func(ctx *armadacontext.Context, job interfaces.LegacySchedulerJob) bool { if job.GetAnnotations() == nil { - ctx.Log.Warnf("can't evict job %s: annotations not initialised", job.GetId()) + ctx.Warnf("can't evict job %s: annotations not initialised", job.GetId()) return false } priorityClassName := job.GetPriorityClassName() @@ -884,7 +885,7 @@ func defaultPostEvictFunc(ctx *armadacontext.Context, job interfaces.LegacySched // Add annotation indicating to the scheduler this this job was evicted. annotations := job.GetAnnotations() if annotations == nil { - ctx.Log.Errorf("error evicting job %s: annotations not initialised", job.GetId()) + ctx.Errorf("error evicting job %s: annotations not initialised", job.GetId()) } else { annotations[schedulerconfig.IsEvictedAnnotation] = "true" } @@ -892,7 +893,7 @@ func defaultPostEvictFunc(ctx *armadacontext.Context, job interfaces.LegacySched // Add node selector ensuring this job is only re-scheduled onto the node it was evicted from. nodeSelector := job.GetNodeSelector() if nodeSelector == nil { - ctx.Log.Errorf("error evicting job %s: nodeSelector not initialised", job.GetId()) + ctx.Errorf("error evicting job %s: nodeSelector not initialised", job.GetId()) } else { nodeSelector[schedulerconfig.NodeIdLabel] = node.Id } diff --git a/internal/scheduler/preempting_queue_scheduler_test.go b/internal/scheduler/preempting_queue_scheduler_test.go index 84538cdccc2..76094017ae5 100644 --- a/internal/scheduler/preempting_queue_scheduler_test.go +++ b/internal/scheduler/preempting_queue_scheduler_test.go @@ -516,7 +516,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { { // Schedule a gang across two nodes. JobsByQueue: map[string][]*jobdb.Job{ - "A": testfixtures.WithGangAnnotationsJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2)), + "A": testfixtures.WithGangAnnotationsJobsAndMinCardinality(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2), 1), }, ExpectedScheduledIndices: map[string][]int{ "A": testfixtures.IntRange(0, 1), diff --git a/internal/scheduler/publisher.go b/internal/scheduler/publisher.go index 0b308141961..598a00fc755 100644 --- a/internal/scheduler/publisher.go +++ b/internal/scheduler/publisher.go @@ -10,10 +10,10 @@ import ( "github.com/gogo/protobuf/proto" "github.com/google/uuid" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/eventutil" + "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/schedulers" "github.com/armadaproject/armada/pkg/armadaevents" ) @@ -103,13 +103,15 @@ func (p *PulsarPublisher) PublishMessages(ctx *armadacontext.Context, events []* // Send messages if shouldPublish() { - log.Debugf("Am leader so will publish") + ctx.Debugf("Am leader so will publish") sendCtx, cancel := armadacontext.WithTimeout(ctx, p.pulsarSendTimeout) errored := false for _, msg := range msgs { p.producer.SendAsync(sendCtx, msg, func(_ pulsar.MessageID, _ *pulsar.ProducerMessage, err error) { if err != nil { - log.WithError(err).Error("error sending message to Pulsar") + logging. + WithStacktrace(ctx, err). + Error("error sending message to Pulsar") errored = true } wg.Done() @@ -121,7 +123,7 @@ func (p *PulsarPublisher) PublishMessages(ctx *armadacontext.Context, events []* return errors.New("One or more messages failed to send to Pulsar") } } else { - log.Debugf("No longer leader so not publishing") + ctx.Debugf("No longer leader so not publishing") } return nil } diff --git a/internal/scheduler/queue_scheduler.go b/internal/scheduler/queue_scheduler.go index cf03c7af3fc..95683904eec 100644 --- a/internal/scheduler/queue_scheduler.go +++ b/internal/scheduler/queue_scheduler.go @@ -63,6 +63,7 @@ func (sch *QueueScheduler) SkipUnsuccessfulSchedulingKeyCheck() { func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResult, error) { nodeIdByJobId := make(map[string]string) scheduledJobs := make([]interfaces.LegacySchedulerJob, 0) + failedJobs := make([]interfaces.LegacySchedulerJob, 0) for { // Peek() returns the next gang to try to schedule. Call Clear() before calling Peek() again. // Calling Clear() after (failing to) schedule ensures we get the next gang in order of smallest fair share. @@ -91,13 +92,21 @@ func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResul if ok, unschedulableReason, err := sch.gangScheduler.Schedule(ctx, gctx); err != nil { return nil, err } else if ok { + // We scheduled the minimum number of gang jobs required. for _, jctx := range gctx.JobSchedulingContexts { - scheduledJobs = append(scheduledJobs, jctx.Job) pctx := jctx.PodSchedulingContext if pctx != nil && pctx.NodeId != "" { + scheduledJobs = append(scheduledJobs, jctx.Job) nodeIdByJobId[jctx.JobId] = pctx.NodeId } } + + // Report any excess gang jobs that failed + for _, jctx := range gctx.JobSchedulingContexts { + if jctx.ShouldFail { + failedJobs = append(failedJobs, jctx.Job) + } + } } else if schedulerconstraints.IsTerminalUnschedulableReason(unschedulableReason) { // If unschedulableReason indicates no more new jobs can be scheduled, // instruct the underlying iterator to only yield evicted jobs from now on. @@ -107,6 +116,7 @@ func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResul // instruct the underlying iterator to only yield evicted jobs for this queue from now on. sch.candidateGangIterator.OnlyYieldEvictedForQueue(gctx.Queue) } + // Clear() to get the next gang in order of smallest fair share. // Calling clear here ensures the gang scheduled in this iteration is accounted for. if err := sch.candidateGangIterator.Clear(); err != nil { @@ -122,6 +132,7 @@ func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResul return &SchedulerResult{ PreemptedJobs: nil, ScheduledJobs: scheduledJobs, + FailedJobs: failedJobs, NodeIdByJobId: nodeIdByJobId, SchedulingContexts: []*schedulercontext.SchedulingContext{sch.schedulingContext}, }, nil @@ -208,6 +219,8 @@ func (it *QueuedGangIterator) Peek() (*schedulercontext.GangSchedulingContext, e Job: job, UnschedulableReason: unsuccessfulJctx.UnschedulableReason, PodSchedulingContext: unsuccessfulJctx.PodSchedulingContext, + // TODO: Move this into gang scheduling context + GangMinCardinality: 1, } if _, err := it.schedulingContext.AddJobSchedulingContext(jctx); err != nil { return nil, err @@ -232,6 +245,7 @@ func (it *QueuedGangIterator) Peek() (*schedulercontext.GangSchedulingContext, e schedulercontext.JobSchedulingContextsFromJobs( it.schedulingContext.PriorityClasses, gang, + GangIdAndCardinalityFromAnnotations, ), ) return it.next, nil @@ -241,6 +255,7 @@ func (it *QueuedGangIterator) Peek() (*schedulercontext.GangSchedulingContext, e schedulercontext.JobSchedulingContextsFromJobs( it.schedulingContext.PriorityClasses, []interfaces.LegacySchedulerJob{job}, + GangIdAndCardinalityFromAnnotations, ), ) return it.next, nil diff --git a/internal/scheduler/queue_scheduler_test.go b/internal/scheduler/queue_scheduler_test.go index 3832db7ceba..ff7c3d0ca20 100644 --- a/internal/scheduler/queue_scheduler_test.go +++ b/internal/scheduler/queue_scheduler_test.go @@ -410,9 +410,9 @@ func TestQueueScheduler(t *testing.T) { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2", configuration.GangMinimumCardinalityAnnotation: "1"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2", configuration.GangMinimumCardinalityAnnotation: "1"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), ), PriorityFactorByQueue: map[string]float64{"A": 1}, ExpectedScheduledIndices: []int{0, 1, 2}, @@ -428,9 +428,9 @@ func TestQueueScheduler(t *testing.T) { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2", configuration.GangMinimumCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2", configuration.GangMinimumCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), ), PriorityFactorByQueue: map[string]float64{"A": 1}, ExpectedScheduledIndices: []int{1}, diff --git a/internal/scheduler/reports_test.go b/internal/scheduler/reports_test.go index fcc0837188a..d989e498fad 100644 --- a/internal/scheduler/reports_test.go +++ b/internal/scheduler/reports_test.go @@ -253,7 +253,7 @@ func withSuccessfulJobSchedulingContext(sctx *schedulercontext.SchedulingContext qctx.SchedulingContext = nil qctx.Created = time.Time{} } - qctx.SuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId} + qctx.SuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId, GangMinCardinality: 1} rl := schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"cpu": resource.MustParse("1")}} qctx.ScheduledResourcesByPriorityClass.AddResourceList("foo", rl) sctx.ScheduledResourcesByPriorityClass.AddResourceList("foo", rl) @@ -293,7 +293,7 @@ func withUnsuccessfulJobSchedulingContext(sctx *schedulercontext.SchedulingConte qctx.SchedulingContext = nil qctx.Created = time.Time{} } - qctx.UnsuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId, UnschedulableReason: "unknown"} + qctx.UnsuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId, UnschedulableReason: "unknown", GangMinCardinality: 1} return sctx } diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index ccc4d998ff5..b23f5ae31ef 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -7,6 +7,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/google/uuid" "github.com/pkg/errors" + "github.com/renstrom/shortuuid" "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/clock" @@ -16,6 +17,7 @@ import ( "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/scheduler/database" + "github.com/armadaproject/armada/internal/scheduler/interfaces" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/kubernetesobjects/affinity" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -116,36 +118,37 @@ func NewScheduler( // Run enters the scheduling loop, which will continue until ctx is cancelled. func (s *Scheduler) Run(ctx *armadacontext.Context) error { - ctx.Log.Infof("starting scheduler with cycle time %s", s.cyclePeriod) - defer ctx.Log.Info("scheduler stopped") + ctx.Infof("starting scheduler with cycle time %s", s.cyclePeriod) + defer ctx.Info("scheduler stopped") // JobDb initialisation. start := s.clock.Now() if err := s.initialise(ctx); err != nil { return err } - ctx.Log.Infof("JobDb initialised in %s", s.clock.Since(start)) + ctx.Infof("JobDb initialised in %s", s.clock.Since(start)) ticker := s.clock.NewTicker(s.cyclePeriod) prevLeaderToken := InvalidLeaderToken() for { select { case <-ctx.Done(): - ctx.Log.Infof("context cancelled; returning.") + ctx.Infof("context cancelled; returning.") return ctx.Err() case <-ticker.C(): start := s.clock.Now() + ctx := armadacontext.WithLogField(ctx, "cycleId", shortuuid.New()) leaderToken := s.leaderController.GetToken() fullUpdate := false - ctx.Log.Infof("received leaderToken; leader status is %t", leaderToken.leader) + ctx.Infof("received leaderToken; leader status is %t", leaderToken.leader) // If we are becoming leader then we must ensure we have caught up to all Pulsar messages if leaderToken.leader && leaderToken != prevLeaderToken { - ctx.Log.Infof("becoming leader") + ctx.Infof("becoming leader") syncContext, cancel := armadacontext.WithTimeout(ctx, 5*time.Minute) err := s.ensureDbUpToDate(syncContext, 1*time.Second) if err != nil { - logging.WithStacktrace(ctx.Log, err).Error("could not become leader") + logging.WithStacktrace(ctx, err).Error("could not become leader") leaderToken = InvalidLeaderToken() } else { fullUpdate = true @@ -165,7 +168,7 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { result, err := s.cycle(ctx, fullUpdate, leaderToken, shouldSchedule) if err != nil { - logging.WithStacktrace(ctx.Log, err).Error("scheduling cycle failure") + logging.WithStacktrace(ctx, err).Error("scheduling cycle failure") leaderToken = InvalidLeaderToken() } @@ -174,13 +177,13 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { s.metrics.ResetGaugeMetrics() if shouldSchedule && leaderToken.leader { - // Only the leader token does real scheduling rounds. + // Only the leader does real scheduling rounds. s.metrics.ReportScheduleCycleTime(cycleTime) - s.metrics.ReportSchedulerResult(result) - ctx.Log.Infof("scheduling cycle completed in %s", cycleTime) + s.metrics.ReportSchedulerResult(ctx, result) + ctx.Infof("scheduling cycle completed in %s", cycleTime) } else { s.metrics.ReportReconcileCycleTime(cycleTime) - ctx.Log.Infof("reconciliation cycle completed in %s", cycleTime) + ctx.Infof("reconciliation cycle completed in %s", cycleTime) } prevLeaderToken = leaderToken @@ -256,7 +259,7 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke if err = s.publisher.PublishMessages(ctx, events, isLeader); err != nil { return } - ctx.Log.Infof("published %d events to pulsar in %s", len(events), s.clock.Since(start)) + ctx.Infof("published %d events to pulsar in %s", len(events), s.clock.Since(start)) txn.Commit() return } @@ -268,7 +271,7 @@ func (s *Scheduler) syncState(ctx *armadacontext.Context) ([]*jobdb.Job, error) if err != nil { return nil, err } - ctx.Log.Infof("received %d updated jobs and %d updated job runs in %s", len(updatedJobs), len(updatedRuns), s.clock.Since(start)) + ctx.Infof("received %d updated jobs and %d updated job runs in %s", len(updatedJobs), len(updatedRuns), s.clock.Since(start)) txn := s.jobDb.WriteTxn() defer txn.Abort() @@ -312,7 +315,7 @@ func (s *Scheduler) syncState(ctx *armadacontext.Context) ([]*jobdb.Job, error) // If the job is nil or terminal at this point then it cannot be active. // In this case we can ignore the run. if job == nil || job.InTerminalState() { - ctx.Log.Debugf("job %s is not active; ignoring update for run %s", jobId, dbRun.RunID) + ctx.Debugf("job %s is not active; ignoring update for run %s", jobId, dbRun.RunID) continue } } @@ -388,7 +391,7 @@ func (s *Scheduler) eventsFromSchedulerResult(result *SchedulerResult) ([]*armad // EventsFromSchedulerResult generates necessary EventSequences from the provided SchedulerResult. func EventsFromSchedulerResult(result *SchedulerResult, time time.Time) ([]*armadaevents.EventSequence, error) { - eventSequences := make([]*armadaevents.EventSequence, 0, len(result.PreemptedJobs)+len(result.ScheduledJobs)) + eventSequences := make([]*armadaevents.EventSequence, 0, len(result.PreemptedJobs)+len(result.ScheduledJobs)+len(result.FailedJobs)) eventSequences, err := AppendEventSequencesFromPreemptedJobs(eventSequences, PreemptedJobsFromSchedulerResult[*jobdb.Job](result), time) if err != nil { return nil, err @@ -397,6 +400,10 @@ func EventsFromSchedulerResult(result *SchedulerResult, time time.Time) ([]*arma if err != nil { return nil, err } + eventSequences, err = AppendEventSequencesFromUnschedulableJobs(eventSequences, result.FailedJobs, time) + if err != nil { + return nil, err + } return eventSequences, nil } @@ -496,6 +503,32 @@ func AppendEventSequencesFromScheduledJobs(eventSequences []*armadaevents.EventS return eventSequences, nil } +func AppendEventSequencesFromUnschedulableJobs(eventSequences []*armadaevents.EventSequence, jobs []interfaces.LegacySchedulerJob, time time.Time) ([]*armadaevents.EventSequence, error) { + for _, job := range jobs { + jobId, err := armadaevents.ProtoUuidFromUlidString(job.GetId()) + if err != nil { + return nil, err + } + gangJobUnschedulableError := &armadaevents.Error{ + Terminal: true, + Reason: &armadaevents.Error_GangJobUnschedulable{GangJobUnschedulable: &armadaevents.GangJobUnschedulable{Message: "Job did not meet the minimum gang cardinality"}}, + } + eventSequences = append(eventSequences, &armadaevents.EventSequence{ + Queue: job.GetQueue(), + JobSetName: job.GetJobSet(), + Events: []*armadaevents.EventSequence_Event{ + { + Created: &time, + Event: &armadaevents.EventSequence_Event_JobErrors{ + JobErrors: &armadaevents.JobErrors{JobId: jobId, Errors: []*armadaevents.Error{gangJobUnschedulableError}}, + }, + }, + }, + }) + } + return eventSequences, nil +} + // generateUpdateMessages generates EventSequences representing the state changes on updated jobs // If there are no state changes then an empty slice will be returned func (s *Scheduler) generateUpdateMessages(ctx *armadacontext.Context, updatedJobs []*jobdb.Job, txn *jobdb.Txn) ([]*armadaevents.EventSequence, error) { @@ -714,14 +747,14 @@ func (s *Scheduler) expireJobsIfNecessary(ctx *armadacontext.Context, txn *jobdb // has been completely removed for executor, heartbeat := range heartbeatTimes { if heartbeat.Before(cutOff) { - ctx.Log.Warnf("Executor %s has not reported a hearbeart since %v. Will expire all jobs running on this executor", executor, heartbeat) + ctx.Warnf("Executor %s has not reported a hearbeart since %v. Will expire all jobs running on this executor", executor, heartbeat) staleExecutors[executor] = true } } // All clusters have had a heartbeat recently. No need to expire any jobs if len(staleExecutors) == 0 { - ctx.Log.Infof("No stale executors found. No jobs need to be expired") + ctx.Infof("No stale executors found. No jobs need to be expired") return nil, nil } @@ -738,7 +771,7 @@ func (s *Scheduler) expireJobsIfNecessary(ctx *armadacontext.Context, txn *jobdb run := job.LatestRun() if run != nil && !job.Queued() && staleExecutors[run.Executor()] { - ctx.Log.Warnf("Cancelling job %s as it is running on lost executor %s", job.Id(), run.Executor()) + ctx.Warnf("Cancelling job %s as it is running on lost executor %s", job.Id(), run.Executor()) jobsToUpdate = append(jobsToUpdate, job.WithQueued(false).WithFailed(true).WithUpdatedRun(run.WithFailed(true))) jobId, err := armadaevents.ProtoUuidFromUlidString(job.Id()) @@ -803,7 +836,7 @@ func (s *Scheduler) initialise(ctx *armadacontext.Context) error { return nil default: if _, err := s.syncState(ctx); err != nil { - ctx.Log.WithError(err).Error("failed to initialise; trying again in 1 second") + logging.WithStacktrace(ctx, err).Error("failed to initialise; trying again in 1 second") time.Sleep(1 * time.Second) } else { // Initialisation succeeded. @@ -830,7 +863,7 @@ func (s *Scheduler) ensureDbUpToDate(ctx *armadacontext.Context, pollInterval ti default: numSent, err = s.publisher.PublishMarkers(ctx, groupId) if err != nil { - ctx.Log.WithError(err).Error("Error sending marker messages to pulsar") + logging.WithStacktrace(ctx, err).Error("Error sending marker messages to pulsar") s.clock.Sleep(pollInterval) } else { messagesSent = true @@ -846,13 +879,15 @@ func (s *Scheduler) ensureDbUpToDate(ctx *armadacontext.Context, pollInterval ti default: numReceived, err := s.jobRepository.CountReceivedPartitions(ctx, groupId) if err != nil { - ctx.Log.WithError(err).Error("Error querying the database or marker messages") + logging. + WithStacktrace(ctx, err). + Error("Error querying the database or marker messages") } if numSent == numReceived { - ctx.Log.Infof("Successfully ensured that database state is up to date") + ctx.Infof("Successfully ensured that database state is up to date") return nil } - ctx.Log.Infof("Recevied %d partitions, still waiting on %d", numReceived, numSent-numReceived) + ctx.Infof("Recevied %d partitions, still waiting on %d", numReceived, numSent-numReceived) s.clock.Sleep(pollInterval) } } diff --git a/internal/scheduler/scheduler_metrics.go b/internal/scheduler/scheduler_metrics.go index 25840fae841..3ba197ebeba 100644 --- a/internal/scheduler/scheduler_metrics.go +++ b/internal/scheduler/scheduler_metrics.go @@ -4,9 +4,9 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/armada/configuration" + "github.com/armadaproject/armada/internal/common/armadacontext" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" "github.com/armadaproject/armada/internal/scheduler/interfaces" ) @@ -157,29 +157,29 @@ func (metrics *SchedulerMetrics) ReportReconcileCycleTime(cycleTime time.Duratio metrics.reconcileCycleTime.Observe(float64(cycleTime.Milliseconds())) } -func (metrics *SchedulerMetrics) ReportSchedulerResult(result SchedulerResult) { +func (metrics *SchedulerMetrics) ReportSchedulerResult(ctx *armadacontext.Context, result SchedulerResult) { if result.EmptyResult { return // TODO: Add logging or maybe place to add failure metric? } // Report the total scheduled jobs (possibly we can get these out of contexts?) - metrics.reportScheduledJobs(result.ScheduledJobs) - metrics.reportPreemptedJobs(result.PreemptedJobs) + metrics.reportScheduledJobs(ctx, result.ScheduledJobs) + metrics.reportPreemptedJobs(ctx, result.PreemptedJobs) // TODO: When more metrics are added, consider consolidating into a single loop over the data. // Report the number of considered jobs. - metrics.reportNumberOfJobsConsidered(result.SchedulingContexts) - metrics.reportQueueShares(result.SchedulingContexts) + metrics.reportNumberOfJobsConsidered(ctx, result.SchedulingContexts) + metrics.reportQueueShares(ctx, result.SchedulingContexts) } -func (metrics *SchedulerMetrics) reportScheduledJobs(scheduledJobs []interfaces.LegacySchedulerJob) { +func (metrics *SchedulerMetrics) reportScheduledJobs(ctx *armadacontext.Context, scheduledJobs []interfaces.LegacySchedulerJob) { jobAggregates := aggregateJobs(scheduledJobs) - observeJobAggregates(metrics.scheduledJobsPerQueue, jobAggregates) + observeJobAggregates(ctx, metrics.scheduledJobsPerQueue, jobAggregates) } -func (metrics *SchedulerMetrics) reportPreemptedJobs(preemptedJobs []interfaces.LegacySchedulerJob) { +func (metrics *SchedulerMetrics) reportPreemptedJobs(ctx *armadacontext.Context, preemptedJobs []interfaces.LegacySchedulerJob) { jobAggregates := aggregateJobs(preemptedJobs) - observeJobAggregates(metrics.preemptedJobsPerQueue, jobAggregates) + observeJobAggregates(ctx, metrics.preemptedJobsPerQueue, jobAggregates) } type collectionKey struct { @@ -200,7 +200,7 @@ func aggregateJobs[S ~[]E, E interfaces.LegacySchedulerJob](scheduledJobs S) map } // observeJobAggregates reports a set of job aggregates to a given CounterVec by queue and priorityClass. -func observeJobAggregates(metric prometheus.CounterVec, jobAggregates map[collectionKey]int) { +func observeJobAggregates(ctx *armadacontext.Context, metric prometheus.CounterVec, jobAggregates map[collectionKey]int) { for key, count := range jobAggregates { queue := key.queue priorityClassName := key.priorityClass @@ -209,14 +209,14 @@ func observeJobAggregates(metric prometheus.CounterVec, jobAggregates map[collec if err != nil { // A metric failure isn't reason to kill the programme. - log.Errorf("error reteriving considered jobs observer for queue %s, priorityClass %s", queue, priorityClassName) + ctx.Errorf("error reteriving considered jobs observer for queue %s, priorityClass %s", queue, priorityClassName) } else { observer.Add(float64(count)) } } } -func (metrics *SchedulerMetrics) reportNumberOfJobsConsidered(schedulingContexts []*schedulercontext.SchedulingContext) { +func (metrics *SchedulerMetrics) reportNumberOfJobsConsidered(ctx *armadacontext.Context, schedulingContexts []*schedulercontext.SchedulingContext) { for _, schedContext := range schedulingContexts { pool := schedContext.Pool for queue, queueContext := range schedContext.QueueSchedulingContexts { @@ -224,7 +224,7 @@ func (metrics *SchedulerMetrics) reportNumberOfJobsConsidered(schedulingContexts observer, err := metrics.consideredJobs.GetMetricWithLabelValues(queue, pool) if err != nil { - log.Errorf("error reteriving considered jobs observer for queue %s, pool %s", queue, pool) + ctx.Errorf("error reteriving considered jobs observer for queue %s, pool %s", queue, pool) } else { observer.Add(float64(count)) } @@ -232,7 +232,7 @@ func (metrics *SchedulerMetrics) reportNumberOfJobsConsidered(schedulingContexts } } -func (metrics *SchedulerMetrics) reportQueueShares(schedulingContexts []*schedulercontext.SchedulingContext) { +func (metrics *SchedulerMetrics) reportQueueShares(ctx *armadacontext.Context, schedulingContexts []*schedulercontext.SchedulingContext) { for _, schedContext := range schedulingContexts { totalCost := schedContext.TotalCost() totalWeight := schedContext.WeightSum @@ -243,7 +243,7 @@ func (metrics *SchedulerMetrics) reportQueueShares(schedulingContexts []*schedul observer, err := metrics.fairSharePerQueue.GetMetricWithLabelValues(queue, pool) if err != nil { - log.Errorf("error reteriving considered jobs observer for queue %s, pool %s", queue, pool) + ctx.Errorf("error retrieving considered jobs observer for queue %s, pool %s", queue, pool) } else { observer.Set(fairShare) } @@ -252,7 +252,7 @@ func (metrics *SchedulerMetrics) reportQueueShares(schedulingContexts []*schedul observer, err = metrics.actualSharePerQueue.GetMetricWithLabelValues(queue, pool) if err != nil { - log.Errorf("error reteriving considered jobs observer for queue %s, pool %s", queue, pool) + ctx.Errorf("error reteriving considered jobs observer for queue %s, pool %s", queue, pool) } else { observer.Set(actualShare) } diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 584f4552d42..2a474d2defd 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -184,6 +184,7 @@ func TestScheduler_TestCycle(t *testing.T) { expectedJobRunLeased []string // ids of jobs we expect to have produced leased messages expectedJobRunErrors []string // ids of jobs we expect to have produced jobRunErrors messages expectedJobErrors []string // ids of jobs we expect to have produced jobErrors messages + expectedJobsToFail []string // ids of jobs we expect to fail without having failed the overall scheduling cycle expectedJobRunPreempted []string // ids of jobs we expect to have produced jobRunPreempted messages expectedJobCancelled []string // ids of jobs we expect to have produced cancelled messages expectedJobReprioritised []string // ids of jobs we expect to have produced reprioritised messages @@ -225,6 +226,12 @@ func TestScheduler_TestCycle(t *testing.T) { expectedQueued: []string{queuedJob.Id()}, expectedQueuedVersion: queuedJob.QueuedVersion(), }, + "FailedJobs in scheduler result will publish appropriate messages": { + initialJobs: []*jobdb.Job{queuedJob}, + expectedJobErrors: []string{queuedJob.Id()}, + expectedJobsToFail: []string{queuedJob.Id()}, + expectedTerminal: []string{queuedJob.Id()}, + }, "No updates to an already leased job": { initialJobs: []*jobdb.Job{leasedJob}, expectedLeased: []string{leasedJob.Id()}, @@ -487,6 +494,7 @@ func TestScheduler_TestCycle(t *testing.T) { schedulingAlgo := &testSchedulingAlgo{ jobsToSchedule: tc.expectedJobRunLeased, jobsToPreempt: tc.expectedJobRunPreempted, + jobsToFail: tc.expectedJobsToFail, shouldError: tc.scheduleError, } publisher := &testPublisher{shouldError: tc.publishError} @@ -998,6 +1006,7 @@ type testSchedulingAlgo struct { numberOfScheduleCalls int jobsToPreempt []string jobsToSchedule []string + jobsToFail []string shouldError bool } @@ -1008,6 +1017,7 @@ func (t *testSchedulingAlgo) Schedule(ctx *armadacontext.Context, txn *jobdb.Txn } preemptedJobs := make([]*jobdb.Job, 0, len(t.jobsToPreempt)) scheduledJobs := make([]*jobdb.Job, 0, len(t.jobsToSchedule)) + failedJobs := make([]*jobdb.Job, 0, len(t.jobsToFail)) for _, id := range t.jobsToPreempt { job := jobDb.GetById(txn, id) if job == nil { @@ -1035,13 +1045,27 @@ func (t *testSchedulingAlgo) Schedule(ctx *armadacontext.Context, txn *jobdb.Txn job = job.WithQueuedVersion(job.QueuedVersion()+1).WithQueued(false).WithNewRun("test-executor", "test-node", "node") scheduledJobs = append(scheduledJobs, job) } + for _, id := range t.jobsToFail { + job := jobDb.GetById(txn, id) + if job == nil { + return nil, errors.Errorf("was asked to lease %s but job does not exist", id) + } + if !job.Queued() { + return nil, errors.Errorf("was asked to lease %s but job was already leased", job.Id()) + } + job = job.WithQueued(false).WithFailed(true) + failedJobs = append(failedJobs, job) + } if err := jobDb.Upsert(txn, preemptedJobs); err != nil { return nil, err } if err := jobDb.Upsert(txn, scheduledJobs); err != nil { return nil, err } - return NewSchedulerResult(preemptedJobs, scheduledJobs, nil), nil + if err := jobDb.Upsert(txn, failedJobs); err != nil { + return nil, err + } + return NewSchedulerResultForTest(preemptedJobs, scheduledJobs, failedJobs, nil), nil } type testPublisher struct { diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 9ba1302c920..c045591b175 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -12,7 +12,6 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -24,6 +23,7 @@ import ( dbcommon "github.com/armadaproject/armada/internal/common/database" grpcCommon "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" + "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/stringinterner" schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" @@ -55,7 +55,7 @@ func Run(config schedulerconfig.Configuration) error { ////////////////////////////////////////////////////////////////////////// // Database setup (postgres and redis) ////////////////////////////////////////////////////////////////////////// - log.Infof("Setting up database connections") + ctx.Infof("Setting up database connections") db, err := dbcommon.OpenPgxPool(config.Postgres) if err != nil { return errors.WithMessage(err, "Error opening connection to postgres") @@ -68,7 +68,9 @@ func Run(config schedulerconfig.Configuration) error { defer func() { err := redisClient.Close() if err != nil { - log.WithError(errors.WithStack(err)).Warnf("Redis client didn't close down cleanly") + logging. + WithStacktrace(ctx, err). + Warnf("Redis client didn't close down cleanly") } }() queueRepository := database.NewLegacyQueueRepository(redisClient) @@ -77,7 +79,7 @@ func Run(config schedulerconfig.Configuration) error { ////////////////////////////////////////////////////////////////////////// // Pulsar ////////////////////////////////////////////////////////////////////////// - log.Infof("Setting up Pulsar connectivity") + ctx.Infof("Setting up Pulsar connectivity") pulsarClient, err := pulsarutils.NewPulsarClient(&config.Pulsar) if err != nil { return errors.WithMessage(err, "Error creating pulsar client") @@ -97,7 +99,7 @@ func Run(config schedulerconfig.Configuration) error { ////////////////////////////////////////////////////////////////////////// // Leader Election ////////////////////////////////////////////////////////////////////////// - leaderController, err := createLeaderController(config.Leader) + leaderController, err := createLeaderController(ctx, config.Leader) if err != nil { return errors.WithMessage(err, "error creating leader controller") } @@ -106,7 +108,7 @@ func Run(config schedulerconfig.Configuration) error { ////////////////////////////////////////////////////////////////////////// // Executor Api ////////////////////////////////////////////////////////////////////////// - log.Infof("Setting up executor api") + ctx.Infof("Setting up executor api") apiProducer, err := pulsarClient.CreateProducer(pulsar.ProducerOptions{ Name: fmt.Sprintf("armada-executor-api-%s", uuid.NewString()), CompressionType: config.Pulsar.CompressionType, @@ -144,7 +146,7 @@ func Run(config schedulerconfig.Configuration) error { } executorapi.RegisterExecutorApiServer(grpcServer, executorServer) services = append(services, func() error { - log.Infof("Executor api listening on %s", lis.Addr()) + ctx.Infof("Executor api listening on %s", lis.Addr()) return grpcServer.Serve(lis) }) services = append(services, grpcCommon.CreateShutdownHandler(ctx, 5*time.Second, grpcServer)) @@ -152,7 +154,7 @@ func Run(config schedulerconfig.Configuration) error { ////////////////////////////////////////////////////////////////////////// // Scheduling ////////////////////////////////////////////////////////////////////////// - log.Infof("setting up scheduling loop") + ctx.Infof("setting up scheduling loop") stringInterner, err := stringinterner.New(config.InternedStringsCacheSize) if err != nil { return errors.WithMessage(err, "error creating string interner") @@ -238,14 +240,14 @@ func Run(config schedulerconfig.Configuration) error { return g.Wait() } -func createLeaderController(config schedulerconfig.LeaderConfig) (LeaderController, error) { +func createLeaderController(ctx *armadacontext.Context, config schedulerconfig.LeaderConfig) (LeaderController, error) { switch mode := strings.ToLower(config.Mode); mode { case "standalone": - log.Infof("Scheduler will run in standalone mode") + ctx.Infof("Scheduler will run in standalone mode") return NewStandaloneLeaderController(), nil case "kubernetes": - log.Infof("Scheduler will run kubernetes mode") - clusterConfig, err := loadClusterConfig() + ctx.Infof("Scheduler will run kubernetes mode") + clusterConfig, err := loadClusterConfig(ctx) if err != nil { return nil, errors.Wrapf(err, "Error creating kubernetes client") } @@ -263,14 +265,14 @@ func createLeaderController(config schedulerconfig.LeaderConfig) (LeaderControll } } -func loadClusterConfig() (*rest.Config, error) { +func loadClusterConfig(ctx *armadacontext.Context) (*rest.Config, error) { config, err := rest.InClusterConfig() if err == rest.ErrNotInCluster { - log.Info("Running with default client configuration") + ctx.Info("Running with default client configuration") rules := clientcmd.NewDefaultClientConfigLoadingRules() overrides := &clientcmd.ConfigOverrides{} return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() } - log.Info("Running with in cluster client configuration") + ctx.Info("Running with in cluster client configuration") return config, err } diff --git a/internal/scheduler/scheduling_algo.go b/internal/scheduler/scheduling_algo.go index a1865d1601b..58be9e48936 100644 --- a/internal/scheduler/scheduling_algo.go +++ b/internal/scheduler/scheduling_algo.go @@ -95,11 +95,12 @@ func (l *FairSchedulingAlgo) Schedule( overallSchedulerResult := &SchedulerResult{ NodeIdByJobId: make(map[string]string), SchedulingContexts: make([]*schedulercontext.SchedulingContext, 0, 0), + FailedJobs: make([]interfaces.LegacySchedulerJob, 0), } // Exit immediately if scheduling is disabled. if l.schedulingConfig.DisableScheduling { - ctx.Log.Info("skipping scheduling - scheduling disabled") + ctx.Info("skipping scheduling - scheduling disabled") return overallSchedulerResult, nil } @@ -121,7 +122,7 @@ func (l *FairSchedulingAlgo) Schedule( select { case <-ctxWithTimeout.Done(): // We've reached the scheduling time limit; exit gracefully. - ctx.Log.Info("ending scheduling round early as we have hit the maximum scheduling duration") + ctx.Info("ending scheduling round early as we have hit the maximum scheduling duration") return overallSchedulerResult, nil default: } @@ -140,7 +141,7 @@ func (l *FairSchedulingAlgo) Schedule( // Assume pool and minimumJobSize are consistent within the group. pool := executorGroup[0].Pool minimumJobSize := executorGroup[0].MinimumJobSize - ctx.Log.Infof( + ctx.Infof( "scheduling on executor group %s with capacity %s", executorGroupLabel, fsctx.totalCapacityByPool[pool].CompactString(), ) @@ -156,30 +157,34 @@ func (l *FairSchedulingAlgo) Schedule( // add the executorGroupLabel back to l.executorGroupsToSchedule such that we try it again next time, // and exit gracefully. l.executorGroupsToSchedule = append(l.executorGroupsToSchedule, executorGroupLabel) - ctx.Log.Info("stopped scheduling early as we have hit the maximum scheduling duration") + ctx.Info("stopped scheduling early as we have hit the maximum scheduling duration") break } else if err != nil { return nil, err } if l.schedulingContextRepository != nil { if err := l.schedulingContextRepository.AddSchedulingContext(sctx); err != nil { - logging.WithStacktrace(ctx.Log, err).Error("failed to add scheduling context") + logging.WithStacktrace(ctx, err).Error("failed to add scheduling context") } } - // Update jobDb. preemptedJobs := PreemptedJobsFromSchedulerResult[*jobdb.Job](schedulerResult) scheduledJobs := ScheduledJobsFromSchedulerResult[*jobdb.Job](schedulerResult) + failedJobs := FailedJobsFromSchedulerResult[*jobdb.Job](schedulerResult) if err := jobDb.Upsert(txn, preemptedJobs); err != nil { return nil, err } if err := jobDb.Upsert(txn, scheduledJobs); err != nil { return nil, err } + if err := jobDb.Upsert(txn, failedJobs); err != nil { + return nil, err + } // Aggregate changes across executors. overallSchedulerResult.PreemptedJobs = append(overallSchedulerResult.PreemptedJobs, schedulerResult.PreemptedJobs...) overallSchedulerResult.ScheduledJobs = append(overallSchedulerResult.ScheduledJobs, schedulerResult.ScheduledJobs...) + overallSchedulerResult.FailedJobs = append(overallSchedulerResult.FailedJobs, schedulerResult.FailedJobs...) overallSchedulerResult.SchedulingContexts = append(overallSchedulerResult.SchedulingContexts, schedulerResult.SchedulingContexts...) maps.Copy(overallSchedulerResult.NodeIdByJobId, schedulerResult.NodeIdByJobId) @@ -459,6 +464,10 @@ func (l *FairSchedulingAlgo) scheduleOnExecutors( result.ScheduledJobs[i] = jobDbJob.WithQueuedVersion(jobDbJob.QueuedVersion()+1).WithQueued(false).WithNewRun(node.Executor, node.Id, node.Name) } } + for i, job := range result.FailedJobs { + jobDbJob := job.(*jobdb.Job) + result.FailedJobs[i] = jobDbJob.WithQueued(false).WithFailed(true) + } return result, sctx, nil } @@ -563,7 +572,9 @@ func (l *FairSchedulingAlgo) filterLaggingExecutors( leasedJobs := leasedJobsByExecutor[executor.Id] executorRuns, err := executor.AllRuns() if err != nil { - logging.WithStacktrace(ctx.Log, err).Errorf("failed to retrieve runs for executor %s; will not be considered for scheduling", executor.Id) + logging. + WithStacktrace(ctx, err). + Errorf("failed to retrieve runs for executor %s; will not be considered for scheduling", executor.Id) continue } executorRunIds := make(map[uuid.UUID]bool, len(executorRuns)) @@ -582,7 +593,7 @@ func (l *FairSchedulingAlgo) filterLaggingExecutors( if numUnacknowledgedJobs <= l.schedulingConfig.MaxUnacknowledgedJobsPerExecutor { activeExecutors = append(activeExecutors, executor) } else { - ctx.Log.Warnf( + ctx.Warnf( "%d unacknowledged jobs on executor %s exceeds limit of %d; executor will not be considered for scheduling", numUnacknowledgedJobs, executor.Id, l.schedulingConfig.MaxUnacknowledgedJobsPerExecutor, ) diff --git a/internal/scheduler/scheduling_algo_test.go b/internal/scheduler/scheduling_algo_test.go index 2bf766ecd40..59487b73f55 100644 --- a/internal/scheduler/scheduling_algo_test.go +++ b/internal/scheduler/scheduling_algo_test.go @@ -46,6 +46,9 @@ func TestSchedule(t *testing.T) { // Indices of queued jobs expected to be scheduled. expectedScheduledIndices []int + + // Count of jobs expected to fail + expectedFailedJobCount int }{ "scheduling": { schedulingConfig: testfixtures.TestSchedulingConfig(), @@ -249,13 +252,21 @@ func TestSchedule(t *testing.T) { }, expectedScheduledIndices: []int{0}, }, - "gang scheduling": { + "gang scheduling successful": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, queues: []*database.Queue{{Name: "A", Weight: 100}}, queuedJobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 2)), expectedScheduledIndices: []int{0, 1}, }, + "gang scheduling successful with some jobs failing to schedule above min cardinality": { + schedulingConfig: testfixtures.TestSchedulingConfig(), + executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + queues: []*database.Queue{{Name: "A", Weight: 100}}, + queuedJobs: testfixtures.WithGangAnnotationsJobsAndMinCardinality(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 10), 2), + expectedScheduledIndices: []int{0, 1}, + expectedFailedJobCount: 8, + }, "not scheduling a gang that does not fit on any executor": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ @@ -433,6 +444,10 @@ func TestSchedule(t *testing.T) { assert.Equal(t, tc.expectedScheduledIndices, actualScheduledIndices) } + // Check that we failed the correct number of excess jobs when a gang schedules >= minimum cardinality + failedJobs := FailedJobsFromSchedulerResult[*jobdb.Job](schedulerResult) + assert.Equal(t, tc.expectedFailedJobCount, len(failedJobs)) + // Check that preempted jobs are marked as such consistently. for _, job := range preemptedJobs { dbJob := jobDb.GetById(txn, job.Id()) @@ -451,6 +466,13 @@ func TestSchedule(t *testing.T) { assert.NotEmpty(t, dbRun.NodeName()) } + // Check that failed jobs are marked as such consistently. + for _, job := range failedJobs { + dbJob := jobDb.GetById(txn, job.Id()) + assert.True(t, dbJob.Failed()) + assert.False(t, dbJob.Queued()) + } + // Check that jobDb was updated correctly. // TODO: Check that there are no unexpected jobs in the jobDb. for _, job := range preemptedJobs { @@ -461,6 +483,10 @@ func TestSchedule(t *testing.T) { dbJob := jobDb.GetById(txn, job.Id()) assert.Equal(t, job, dbJob) } + for _, job := range failedJobs { + dbJob := jobDb.GetById(txn, job.Id()) + assert.Equal(t, job, dbJob) + } }) } } diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 94fa8989b84..1c282e8c303 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -534,6 +534,10 @@ func (s *Simulator) handleScheduleEvent() error { if err != nil { return err } + eventSequences, err = scheduler.AppendEventSequencesFromUnschedulableJobs(eventSequences, result.FailedJobs, s.time) + if err != nil { + return err + } } } txn.Commit() diff --git a/internal/scheduler/submitcheck.go b/internal/scheduler/submitcheck.go index bf79e0eb317..4fe71c9c59f 100644 --- a/internal/scheduler/submitcheck.go +++ b/internal/scheduler/submitcheck.go @@ -8,12 +8,12 @@ import ( lru "github.com/hashicorp/golang-lru" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "golang.org/x/exp/maps" "k8s.io/apimachinery/pkg/util/clock" "github.com/armadaproject/armada/internal/armada/configuration" "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/internal/common/logging" armadaslices "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/common/types" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" @@ -101,7 +101,9 @@ func (srv *SubmitChecker) Run(ctx *armadacontext.Context) error { func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { executors, err := srv.executorRepository.GetExecutors(ctx) if err != nil { - log.WithError(err).Error("Error fetching executors") + logging. + WithStacktrace(ctx, err). + Error("Error fetching executors") return } for _, executor := range executors { @@ -114,10 +116,14 @@ func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { } srv.mu.Unlock() if err != nil { - log.WithError(err).Errorf("Error constructing node db for executor %s", executor.Id) + logging. + WithStacktrace(ctx, err). + Errorf("Error constructing node db for executor %s", executor.Id) } } else { - log.WithError(err).Warnf("Error clearing nodedb for executor %s", executor.Id) + logging. + WithStacktrace(ctx, err). + Warnf("Error clearing nodedb for executor %s", executor.Id) } } @@ -128,17 +134,21 @@ func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { } func (srv *SubmitChecker) CheckApiJobs(jobs []*api.Job) (bool, string) { - return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs)) + return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs, GangIdAndCardinalityFromAnnotations)) } func (srv *SubmitChecker) CheckJobDbJobs(jobs []*jobdb.Job) (bool, string) { - return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs)) + return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs, GangIdAndCardinalityFromAnnotations)) } func (srv *SubmitChecker) check(jctxs []*schedulercontext.JobSchedulingContext) (bool, string) { // First, check if all jobs can be scheduled individually. for i, jctx := range jctxs { + // Override min cardinality to enable individual job scheduling checks, but reset after + originalGangMinCardinality := jctx.GangMinCardinality + jctx.GangMinCardinality = 1 schedulingResult := srv.getIndividualSchedulingResult(jctx) + jctx.GangMinCardinality = originalGangMinCardinality if !schedulingResult.isSchedulable { return schedulingResult.isSchedulable, fmt.Sprintf("%d-th job unschedulable:\n%s", i, schedulingResult.reason) } @@ -241,7 +251,7 @@ func (srv *SubmitChecker) getSchedulingResult(jctxs []*schedulercontext.JobSched sb.WriteString("\n") } else { sb.WriteString(":") - sb.WriteString(fmt.Sprintf(" %d out of %d pods schedulable\n", numSuccessfullyScheduled, len(jctxs))) + sb.WriteString(fmt.Sprintf(" %d out of %d pods schedulable (minCardinality %d)\n", numSuccessfullyScheduled, len(jctxs), jctxs[0].GangMinCardinality)) } } return schedulingResult{isSchedulable: isSchedulable, reason: sb.String()} diff --git a/internal/scheduler/submitcheck_test.go b/internal/scheduler/submitcheck_test.go index 87be5674bf8..726e91fa6c7 100644 --- a/internal/scheduler/submitcheck_test.go +++ b/internal/scheduler/submitcheck_test.go @@ -1,6 +1,7 @@ package scheduler import ( + "fmt" "testing" "time" @@ -218,7 +219,10 @@ func testNJobGang(n int) []*api.Job { gang := make([]*api.Job, n) for i := 0; i < n; i++ { job := test1CoreCpuJob() - job.Annotations = map[string]string{configuration.GangIdAnnotation: gangId} + job.Annotations = map[string]string{ + configuration.GangIdAnnotation: gangId, + configuration.GangCardinalityAnnotation: fmt.Sprintf("%d", n), + configuration.GangMinimumCardinalityAnnotation: fmt.Sprintf("%d", n)} gang[i] = job } return gang diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index e73d246c74a..8592ed06b89 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -321,7 +321,17 @@ func WithGangAnnotationsJobs(jobs []*jobdb.Job) []*jobdb.Job { gangId := uuid.NewString() gangCardinality := fmt.Sprintf("%d", len(jobs)) return WithAnnotationsJobs( - map[string]string{configuration.GangIdAnnotation: gangId, configuration.GangCardinalityAnnotation: gangCardinality}, + map[string]string{configuration.GangIdAnnotation: gangId, configuration.GangCardinalityAnnotation: gangCardinality, configuration.GangMinimumCardinalityAnnotation: gangCardinality}, + jobs, + ) +} + +func WithGangAnnotationsJobsAndMinCardinality(jobs []*jobdb.Job, minimumCardinality int) []*jobdb.Job { + gangId := uuid.NewString() + gangCardinality := fmt.Sprintf("%d", len(jobs)) + gangMinCardinality := fmt.Sprintf("%d", minimumCardinality) + return WithAnnotationsJobs( + map[string]string{configuration.GangIdAnnotation: gangId, configuration.GangCardinalityAnnotation: gangCardinality, configuration.GangMinimumCardinalityAnnotation: gangMinCardinality}, jobs, ) } diff --git a/magefiles/linting.go b/magefiles/linting.go index bc7094cebff..e301850e93c 100644 --- a/magefiles/linting.go +++ b/magefiles/linting.go @@ -63,7 +63,7 @@ func LintFix() error { } // Linting Check -func CheckLint() error { +func LintCheck() error { mg.Deps(golangciLintCheck) cmd, err := go_TEST_CMD() if err != nil { diff --git a/pkg/armadaevents/events.pb.go b/pkg/armadaevents/events.pb.go index 2e3fe18f078..ea9d8874f21 100644 --- a/pkg/armadaevents/events.pb.go +++ b/pkg/armadaevents/events.pb.go @@ -2364,6 +2364,7 @@ type Error struct { // *Error_PodLeaseReturned // *Error_PodTerminated // *Error_JobRunPreemptedError + // *Error_GangJobUnschedulable Reason isError_Reason `protobuf_oneof:"reason"` } @@ -2436,6 +2437,9 @@ type Error_PodTerminated struct { type Error_JobRunPreemptedError struct { JobRunPreemptedError *JobRunPreemptedError `protobuf:"bytes,11,opt,name=jobRunPreemptedError,proto3,oneof" json:"jobRunPreemptedError,omitempty"` } +type Error_GangJobUnschedulable struct { + GangJobUnschedulable *GangJobUnschedulable `protobuf:"bytes,12,opt,name=gangJobUnschedulable,proto3,oneof" json:"gangJobUnschedulable,omitempty"` +} func (*Error_KubernetesError) isError_Reason() {} func (*Error_ContainerError) isError_Reason() {} @@ -2447,6 +2451,7 @@ func (*Error_PodError) isError_Reason() {} func (*Error_PodLeaseReturned) isError_Reason() {} func (*Error_PodTerminated) isError_Reason() {} func (*Error_JobRunPreemptedError) isError_Reason() {} +func (*Error_GangJobUnschedulable) isError_Reason() {} func (m *Error) GetReason() isError_Reason { if m != nil { @@ -2532,6 +2537,13 @@ func (m *Error) GetJobRunPreemptedError() *JobRunPreemptedError { return nil } +func (m *Error) GetGangJobUnschedulable() *GangJobUnschedulable { + if x, ok := m.GetReason().(*Error_GangJobUnschedulable); ok { + return x.GangJobUnschedulable + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*Error) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -2545,6 +2557,7 @@ func (*Error) XXX_OneofWrappers() []interface{} { (*Error_PodLeaseReturned)(nil), (*Error_PodTerminated)(nil), (*Error_JobRunPreemptedError)(nil), + (*Error_GangJobUnschedulable)(nil), } } @@ -3131,6 +3144,50 @@ func (m *JobRunPreemptedError) XXX_DiscardUnknown() { var xxx_messageInfo_JobRunPreemptedError proto.InternalMessageInfo +type GangJobUnschedulable struct { + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (m *GangJobUnschedulable) Reset() { *m = GangJobUnschedulable{} } +func (m *GangJobUnschedulable) String() string { return proto.CompactTextString(m) } +func (*GangJobUnschedulable) ProtoMessage() {} +func (*GangJobUnschedulable) Descriptor() ([]byte, []int) { + return fileDescriptor_6aab92ca59e015f8, []int{38} +} +func (m *GangJobUnschedulable) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GangJobUnschedulable) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GangJobUnschedulable.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GangJobUnschedulable) XXX_Merge(src proto.Message) { + xxx_messageInfo_GangJobUnschedulable.Merge(m, src) +} +func (m *GangJobUnschedulable) XXX_Size() int { + return m.Size() +} +func (m *GangJobUnschedulable) XXX_DiscardUnknown() { + xxx_messageInfo_GangJobUnschedulable.DiscardUnknown(m) +} + +var xxx_messageInfo_GangJobUnschedulable proto.InternalMessageInfo + +func (m *GangJobUnschedulable) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + // Generated by the scheduler whenever it detects a SubmitJob message that includes a previously used deduplication id // (i.e., when it detects a duplicate job submission). type JobDuplicateDetected struct { @@ -3142,7 +3199,7 @@ func (m *JobDuplicateDetected) Reset() { *m = JobDuplicateDetected{} } func (m *JobDuplicateDetected) String() string { return proto.CompactTextString(m) } func (*JobDuplicateDetected) ProtoMessage() {} func (*JobDuplicateDetected) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{38} + return fileDescriptor_6aab92ca59e015f8, []int{39} } func (m *JobDuplicateDetected) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3201,7 +3258,7 @@ func (m *JobRunPreempted) Reset() { *m = JobRunPreempted{} } func (m *JobRunPreempted) String() string { return proto.CompactTextString(m) } func (*JobRunPreempted) ProtoMessage() {} func (*JobRunPreempted) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{39} + return fileDescriptor_6aab92ca59e015f8, []int{40} } func (m *JobRunPreempted) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3270,7 +3327,7 @@ func (m *PartitionMarker) Reset() { *m = PartitionMarker{} } func (m *PartitionMarker) String() string { return proto.CompactTextString(m) } func (*PartitionMarker) ProtoMessage() {} func (*PartitionMarker) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{40} + return fileDescriptor_6aab92ca59e015f8, []int{41} } func (m *PartitionMarker) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3323,7 +3380,7 @@ func (m *JobRunPreemptionRequested) Reset() { *m = JobRunPreemptionReque func (m *JobRunPreemptionRequested) String() string { return proto.CompactTextString(m) } func (*JobRunPreemptionRequested) ProtoMessage() {} func (*JobRunPreemptionRequested) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{41} + return fileDescriptor_6aab92ca59e015f8, []int{42} } func (m *JobRunPreemptionRequested) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3414,6 +3471,7 @@ func init() { proto.RegisterType((*LeaseExpired)(nil), "armadaevents.LeaseExpired") proto.RegisterType((*MaxRunsExceeded)(nil), "armadaevents.MaxRunsExceeded") proto.RegisterType((*JobRunPreemptedError)(nil), "armadaevents.JobRunPreemptedError") + proto.RegisterType((*GangJobUnschedulable)(nil), "armadaevents.GangJobUnschedulable") proto.RegisterType((*JobDuplicateDetected)(nil), "armadaevents.JobDuplicateDetected") proto.RegisterType((*JobRunPreempted)(nil), "armadaevents.JobRunPreempted") proto.RegisterType((*PartitionMarker)(nil), "armadaevents.PartitionMarker") @@ -3423,222 +3481,224 @@ func init() { func init() { proto.RegisterFile("pkg/armadaevents/events.proto", fileDescriptor_6aab92ca59e015f8) } var fileDescriptor_6aab92ca59e015f8 = []byte{ - // 3431 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5b, 0x4b, 0x6c, 0x1b, 0xc7, - 0xf9, 0xf7, 0x92, 0x12, 0x1f, 0x1f, 0x25, 0x91, 0x1e, 0xcb, 0x0a, 0xad, 0xd8, 0xa2, 0xb3, 0xce, - 0xff, 0x1f, 0x27, 0x48, 0xc8, 0xc4, 0x79, 0x20, 0x8f, 0x22, 0x81, 0x68, 0x2b, 0xb1, 0x1d, 0xcb, - 0x76, 0x28, 0x3b, 0x75, 0x83, 0x14, 0xcc, 0x92, 0x3b, 0xa2, 0xd6, 0x22, 0x77, 0x37, 0xfb, 0x90, - 0x25, 0x20, 0x87, 0xb6, 0x68, 0xd3, 0x5b, 0x6b, 0xa0, 0x3d, 0x14, 0xe8, 0x21, 0xbd, 0x36, 0x40, - 0x6f, 0x05, 0x7a, 0xee, 0x2d, 0x05, 0x8a, 0x22, 0xed, 0xa9, 0x27, 0xb6, 0x48, 0xd0, 0x43, 0x79, - 0xe8, 0xb9, 0xed, 0xa9, 0x98, 0xd7, 0xee, 0xcc, 0xee, 0xd2, 0x56, 0xfc, 0xa8, 0x53, 0xf8, 0x24, - 0xed, 0xef, 0x7b, 0xcd, 0xce, 0xcc, 0xf7, 0xed, 0xf7, 0x7d, 0x33, 0x84, 0x63, 0xee, 0xf6, 0xa0, - 0x65, 0x78, 0x23, 0xc3, 0x34, 0xf0, 0x0e, 0xb6, 0x03, 0xbf, 0xc5, 0xfe, 0x34, 0x5d, 0xcf, 0x09, - 0x1c, 0x34, 0x27, 0x93, 0x96, 0xf5, 0xed, 0x97, 0xfd, 0xa6, 0xe5, 0xb4, 0x0c, 0xd7, 0x6a, 0xf5, - 0x1d, 0x0f, 0xb7, 0x76, 0x9e, 0x6b, 0x0d, 0xb0, 0x8d, 0x3d, 0x23, 0xc0, 0x26, 0x93, 0x58, 0x3e, - 0x29, 0xf1, 0xd8, 0x38, 0xb8, 0xe1, 0x78, 0xdb, 0x96, 0x3d, 0xc8, 0xe2, 0x6c, 0x0c, 0x1c, 0x67, - 0x30, 0xc4, 0x2d, 0xfa, 0xd4, 0x0b, 0x37, 0x5b, 0x81, 0x35, 0xc2, 0x7e, 0x60, 0x8c, 0x5c, 0xce, - 0xf0, 0x42, 0xac, 0x6a, 0x64, 0xf4, 0xb7, 0x2c, 0x1b, 0x7b, 0x7b, 0x2d, 0x3a, 0x5e, 0xd7, 0x6a, - 0x79, 0xd8, 0x77, 0x42, 0xaf, 0x8f, 0x53, 0x6a, 0x9f, 0x19, 0x58, 0xc1, 0x56, 0xd8, 0x6b, 0xf6, - 0x9d, 0x51, 0x6b, 0xe0, 0x0c, 0x9c, 0x58, 0x3f, 0x79, 0xa2, 0x0f, 0xf4, 0x3f, 0xce, 0xfe, 0xaa, - 0x65, 0x07, 0xd8, 0xb3, 0x8d, 0x61, 0xcb, 0xef, 0x6f, 0x61, 0x33, 0x1c, 0x62, 0x2f, 0xfe, 0xcf, - 0xe9, 0x5d, 0xc7, 0xfd, 0xc0, 0x4f, 0x01, 0x4c, 0x56, 0xbf, 0xb9, 0x08, 0xf3, 0x6b, 0x64, 0x6a, - 0x36, 0xf0, 0x87, 0x21, 0xb6, 0xfb, 0x18, 0x3d, 0x09, 0xb3, 0x1f, 0x86, 0x38, 0xc4, 0x75, 0xed, - 0xb8, 0x76, 0xb2, 0xdc, 0x3e, 0x34, 0x19, 0x37, 0xaa, 0x14, 0x78, 0xda, 0x19, 0x59, 0x01, 0x1e, - 0xb9, 0xc1, 0x5e, 0x87, 0x71, 0xa0, 0x57, 0x61, 0xee, 0xba, 0xd3, 0xeb, 0xfa, 0x38, 0xe8, 0xda, - 0xc6, 0x08, 0xd7, 0x73, 0x54, 0xa2, 0x3e, 0x19, 0x37, 0x16, 0xaf, 0x3b, 0xbd, 0x0d, 0x1c, 0x5c, - 0x34, 0x46, 0xb2, 0x18, 0xc4, 0x28, 0x7a, 0x06, 0x8a, 0xa1, 0x8f, 0xbd, 0xae, 0x65, 0xd6, 0xf3, - 0x54, 0x6c, 0x71, 0x32, 0x6e, 0xd4, 0x08, 0x74, 0xce, 0x94, 0x44, 0x0a, 0x0c, 0x41, 0x4f, 0x43, - 0x61, 0xe0, 0x39, 0xa1, 0xeb, 0xd7, 0x67, 0x8e, 0xe7, 0x05, 0x37, 0x43, 0x64, 0x6e, 0x86, 0xa0, - 0x4b, 0x50, 0x60, 0xeb, 0x5d, 0x9f, 0x3d, 0x9e, 0x3f, 0x59, 0x39, 0xf5, 0x58, 0x53, 0xde, 0x04, - 0x4d, 0xe5, 0x85, 0xd9, 0x13, 0x53, 0xc8, 0xe8, 0xb2, 0x42, 0xbe, 0x6d, 0xfe, 0x7e, 0x10, 0x66, - 0x29, 0x1f, 0xba, 0x04, 0xc5, 0xbe, 0x87, 0xc9, 0x62, 0xd5, 0xd1, 0x71, 0xed, 0x64, 0xe5, 0xd4, - 0x72, 0x93, 0x6d, 0x82, 0xa6, 0x58, 0xa4, 0xe6, 0x15, 0xb1, 0x09, 0xda, 0x47, 0x26, 0xe3, 0xc6, - 0x41, 0xce, 0x1e, 0x6b, 0xbd, 0xf9, 0x97, 0x86, 0xd6, 0x11, 0x5a, 0xd0, 0x65, 0x28, 0xfb, 0x61, - 0x6f, 0x64, 0x05, 0xe7, 0x9d, 0x1e, 0x9d, 0xf3, 0xca, 0xa9, 0x47, 0xd4, 0xe1, 0x6e, 0x08, 0x72, - 0xfb, 0x91, 0xc9, 0xb8, 0x71, 0x28, 0xe2, 0x8e, 0x35, 0x9e, 0x3d, 0xd0, 0x89, 0x95, 0xa0, 0x2d, - 0xa8, 0x7a, 0xd8, 0xf5, 0x2c, 0xc7, 0xb3, 0x02, 0xcb, 0xc7, 0x44, 0x6f, 0x8e, 0xea, 0x3d, 0xa6, - 0xea, 0xed, 0xa8, 0x4c, 0xed, 0x63, 0x93, 0x71, 0xe3, 0x48, 0x42, 0x52, 0xb1, 0x91, 0x54, 0x8b, - 0x02, 0x40, 0x09, 0x68, 0x03, 0x07, 0x74, 0x3d, 0x2b, 0xa7, 0x8e, 0xdf, 0xd2, 0xd8, 0x06, 0x0e, - 0xda, 0xc7, 0x27, 0xe3, 0xc6, 0xd1, 0xb4, 0xbc, 0x62, 0x32, 0x43, 0x3f, 0x1a, 0x42, 0x4d, 0x46, - 0x4d, 0xf2, 0x82, 0x33, 0xd4, 0xe6, 0xca, 0x74, 0x9b, 0x84, 0xab, 0xbd, 0x32, 0x19, 0x37, 0x96, - 0x93, 0xb2, 0x8a, 0xbd, 0x94, 0x66, 0xb2, 0x3e, 0x7d, 0xc3, 0xee, 0xe3, 0x21, 0x31, 0x33, 0x9b, - 0xb5, 0x3e, 0xa7, 0x05, 0x99, 0xad, 0x4f, 0xc4, 0xad, 0xae, 0x4f, 0x04, 0xa3, 0xf7, 0x61, 0x2e, - 0x7a, 0x20, 0xf3, 0x55, 0xe0, 0xfb, 0x28, 0x5b, 0x29, 0x99, 0xa9, 0xe5, 0xc9, 0xb8, 0xb1, 0x24, - 0xcb, 0x28, 0xaa, 0x15, 0x6d, 0xb1, 0xf6, 0x21, 0x9b, 0x99, 0xe2, 0x74, 0xed, 0x8c, 0x43, 0xd6, - 0x3e, 0x4c, 0xcf, 0x88, 0xa2, 0x8d, 0x68, 0x27, 0x4e, 0x1c, 0xf6, 0xfb, 0x18, 0x9b, 0xd8, 0xac, - 0x97, 0xb2, 0xb4, 0x9f, 0x97, 0x38, 0x98, 0x76, 0x59, 0x46, 0xd5, 0x2e, 0x53, 0xc8, 0x5c, 0x5f, - 0x77, 0x7a, 0x6b, 0x9e, 0xe7, 0x78, 0x7e, 0xbd, 0x9c, 0x35, 0xd7, 0xe7, 0x05, 0x99, 0xcd, 0x75, - 0xc4, 0xad, 0xce, 0x75, 0x04, 0xf3, 0xf1, 0x76, 0x42, 0xfb, 0x02, 0x36, 0x7c, 0x6c, 0xd6, 0x61, - 0xca, 0x78, 0x23, 0x8e, 0x68, 0xbc, 0x11, 0x92, 0x1a, 0x6f, 0x44, 0x41, 0x26, 0x2c, 0xb0, 0xe7, - 0x55, 0xdf, 0xb7, 0x06, 0x36, 0x36, 0xeb, 0x15, 0xaa, 0xff, 0x68, 0x96, 0x7e, 0xc1, 0xd3, 0x3e, - 0x3a, 0x19, 0x37, 0xea, 0xaa, 0x9c, 0x62, 0x23, 0xa1, 0x13, 0x7d, 0x00, 0xf3, 0x0c, 0xe9, 0x84, - 0xb6, 0x6d, 0xd9, 0x83, 0xfa, 0x1c, 0x35, 0xf2, 0x68, 0x96, 0x11, 0xce, 0xd2, 0x7e, 0x74, 0x32, - 0x6e, 0x3c, 0xa2, 0x48, 0x29, 0x26, 0x54, 0x85, 0x24, 0x62, 0x30, 0x20, 0x5e, 0xd8, 0xf9, 0xac, - 0x88, 0x71, 0x5e, 0x65, 0x62, 0x11, 0x23, 0x21, 0xa9, 0x46, 0x8c, 0x04, 0x31, 0x5e, 0x0f, 0xbe, - 0xc8, 0x0b, 0xd3, 0xd7, 0x83, 0xaf, 0xb3, 0xb4, 0x1e, 0x19, 0x4b, 0xad, 0x68, 0x43, 0x1f, 0x01, - 0xf9, 0xf0, 0x9c, 0x09, 0xdd, 0xa1, 0xd5, 0x37, 0x02, 0x7c, 0x06, 0x07, 0xb8, 0x4f, 0x22, 0x75, - 0x95, 0x5a, 0xd1, 0x53, 0x56, 0x52, 0x9c, 0x6d, 0x7d, 0x32, 0x6e, 0xac, 0x64, 0xe9, 0x50, 0xac, - 0x66, 0x5a, 0x41, 0xdf, 0xd1, 0xe0, 0xb0, 0x1f, 0x18, 0xb6, 0x69, 0x0c, 0x1d, 0x1b, 0x9f, 0xb3, - 0x07, 0x1e, 0xf6, 0xfd, 0x73, 0xf6, 0xa6, 0x53, 0xaf, 0x51, 0xfb, 0x27, 0x12, 0x61, 0x3d, 0x8b, - 0xb5, 0x7d, 0x62, 0x32, 0x6e, 0x34, 0x32, 0xb5, 0x28, 0x23, 0xc8, 0x36, 0x84, 0x76, 0xe1, 0x90, - 0xc8, 0x2a, 0xae, 0x06, 0xd6, 0xd0, 0xf2, 0x8d, 0xc0, 0x72, 0xec, 0xfa, 0x41, 0x6a, 0xff, 0xb1, - 0x64, 0x74, 0x4c, 0x31, 0xb6, 0x1f, 0x9b, 0x8c, 0x1b, 0xc7, 0x32, 0x34, 0x28, 0xb6, 0xb3, 0x4c, - 0xc4, 0x5b, 0xe8, 0xb2, 0x87, 0x09, 0x23, 0x36, 0xeb, 0x87, 0xa6, 0x6f, 0xa1, 0x88, 0x49, 0xde, - 0x42, 0x11, 0x98, 0xb5, 0x85, 0x22, 0x22, 0xb1, 0xe4, 0x1a, 0x5e, 0x60, 0x11, 0xb3, 0xeb, 0x86, - 0xb7, 0x8d, 0xbd, 0xfa, 0x62, 0x96, 0xa5, 0xcb, 0x2a, 0x13, 0xb3, 0x94, 0x90, 0x54, 0x2d, 0x25, - 0x88, 0xe8, 0xa6, 0x06, 0xea, 0xd0, 0x2c, 0xc7, 0xee, 0x90, 0xb4, 0xc1, 0x27, 0xaf, 0x77, 0x98, - 0x1a, 0x7d, 0xe2, 0x16, 0xaf, 0x27, 0xb3, 0xb7, 0x9f, 0x98, 0x8c, 0x1b, 0x27, 0xa6, 0x6a, 0x53, - 0x06, 0x32, 0xdd, 0x28, 0xba, 0x06, 0x15, 0x42, 0xc4, 0x34, 0x01, 0x33, 0xeb, 0x4b, 0x74, 0x0c, - 0x47, 0xd2, 0x63, 0xe0, 0x0c, 0x34, 0x03, 0x39, 0x2c, 0x49, 0x28, 0x76, 0x64, 0x55, 0xed, 0x22, - 0xcc, 0x52, 0x79, 0x7d, 0x52, 0x80, 0x43, 0x19, 0x7b, 0x03, 0xbd, 0x0e, 0x05, 0x2f, 0xb4, 0x49, - 0xc2, 0xc6, 0xb2, 0x14, 0xa4, 0x5a, 0xbd, 0x1a, 0x5a, 0x26, 0xcb, 0x16, 0xbd, 0xd0, 0x56, 0x72, - 0xb8, 0x59, 0x0a, 0x10, 0x79, 0x92, 0x2d, 0x5a, 0x26, 0xcf, 0x46, 0xa6, 0xca, 0x5f, 0x77, 0x7a, - 0xaa, 0x3c, 0x05, 0x10, 0x86, 0x79, 0xb1, 0xf1, 0xba, 0x16, 0xf1, 0x2a, 0x96, 0x67, 0x3c, 0xae, - 0xaa, 0x79, 0x3b, 0xec, 0x61, 0xcf, 0xc6, 0x01, 0xf6, 0xc5, 0x3b, 0x50, 0xb7, 0xa2, 0x51, 0xc4, - 0x93, 0x10, 0x49, 0xff, 0x9c, 0x8c, 0xa3, 0x9f, 0x6a, 0x50, 0x1f, 0x19, 0xbb, 0x5d, 0x01, 0xfa, - 0xdd, 0x4d, 0xc7, 0xeb, 0xba, 0xd8, 0xb3, 0x1c, 0x93, 0x26, 0x9f, 0x95, 0x53, 0xdf, 0xb8, 0xad, - 0x23, 0x35, 0xd7, 0x8d, 0x5d, 0x01, 0xfb, 0x6f, 0x3a, 0xde, 0x65, 0x2a, 0xbe, 0x66, 0x07, 0xde, - 0x5e, 0xfb, 0xd8, 0x67, 0xe3, 0xc6, 0x01, 0xb2, 0x2c, 0xa3, 0x2c, 0x9e, 0x4e, 0x36, 0x8c, 0x7e, - 0xac, 0xc1, 0x52, 0xe0, 0x04, 0xc6, 0xb0, 0xdb, 0x0f, 0x47, 0xe1, 0xd0, 0x08, 0xac, 0x1d, 0xdc, - 0x0d, 0x7d, 0x63, 0x80, 0x79, 0x8e, 0xfb, 0xda, 0xed, 0x07, 0x75, 0x85, 0xc8, 0x9f, 0x8e, 0xc4, - 0xaf, 0x12, 0x69, 0x36, 0xa6, 0xa3, 0x7c, 0x4c, 0x8b, 0x41, 0x06, 0x4b, 0x27, 0x13, 0x5d, 0xfe, - 0x85, 0x06, 0xcb, 0xd3, 0x5f, 0x13, 0x9d, 0x80, 0xfc, 0x36, 0xde, 0xe3, 0x55, 0xc4, 0xc1, 0xc9, - 0xb8, 0x31, 0xbf, 0x8d, 0xf7, 0xa4, 0x59, 0x27, 0x54, 0xf4, 0x2d, 0x98, 0xdd, 0x31, 0x86, 0x21, - 0xe6, 0x5b, 0xa2, 0xd9, 0x64, 0xf5, 0x52, 0x53, 0xae, 0x97, 0x9a, 0xee, 0xf6, 0x80, 0x00, 0x4d, - 0xb1, 0x22, 0xcd, 0x77, 0x42, 0xc3, 0x0e, 0xac, 0x60, 0x8f, 0x6d, 0x17, 0xaa, 0x40, 0xde, 0x2e, - 0x14, 0x78, 0x35, 0xf7, 0xb2, 0xb6, 0xfc, 0x89, 0x06, 0x47, 0xa6, 0xbe, 0xf4, 0xd7, 0x61, 0x84, - 0x7a, 0x17, 0x66, 0xc8, 0xc6, 0x27, 0xf5, 0xcd, 0x96, 0x35, 0xd8, 0x7a, 0xe9, 0x05, 0x3a, 0x9c, - 0x02, 0x2b, 0x47, 0x18, 0x22, 0x97, 0x23, 0x0c, 0x21, 0x35, 0xda, 0xd0, 0xb9, 0xf1, 0xd2, 0x0b, - 0x74, 0x50, 0x05, 0x66, 0x84, 0x02, 0xb2, 0x11, 0x0a, 0xe8, 0xbf, 0x2e, 0x40, 0x39, 0x2a, 0x20, - 0x24, 0x1f, 0xd4, 0xee, 0xc8, 0x07, 0xcf, 0x42, 0xcd, 0xc4, 0x26, 0xff, 0xf2, 0x59, 0x8e, 0x2d, - 0xbc, 0xb9, 0xcc, 0xa2, 0xab, 0x42, 0x53, 0xe4, 0xab, 0x09, 0x12, 0x3a, 0x05, 0x25, 0x9e, 0x68, - 0xef, 0x51, 0x47, 0x9e, 0x6f, 0x2f, 0x4d, 0xc6, 0x0d, 0x24, 0x30, 0x49, 0x34, 0xe2, 0x43, 0x1d, - 0x00, 0x56, 0xbd, 0xae, 0xe3, 0xc0, 0xe0, 0x29, 0x7f, 0x5d, 0x7d, 0x83, 0x4b, 0x11, 0x9d, 0xd5, - 0xa1, 0x31, 0xbf, 0x5c, 0x87, 0xc6, 0x28, 0x7a, 0x1f, 0x60, 0x64, 0x58, 0x36, 0x93, 0xe3, 0xf9, - 0xbd, 0x3e, 0x2d, 0xa4, 0xac, 0x47, 0x9c, 0x4c, 0x7b, 0x2c, 0x29, 0x6b, 0x8f, 0x51, 0x52, 0x2d, - 0xf2, 0x7a, 0xbb, 0x5e, 0xa0, 0x5e, 0xba, 0x32, 0x4d, 0x35, 0x57, 0x7b, 0x98, 0x54, 0x8c, 0x5c, - 0x44, 0xd2, 0x29, 0xb4, 0x90, 0x69, 0x1b, 0x5a, 0x9b, 0x38, 0xb0, 0x46, 0x98, 0x66, 0xf6, 0x7c, - 0xda, 0x04, 0x26, 0x4f, 0x9b, 0xc0, 0xd0, 0xcb, 0x00, 0x46, 0xb0, 0xee, 0xf8, 0xc1, 0x25, 0xbb, - 0x8f, 0x69, 0xc6, 0x5e, 0x62, 0xc3, 0x8f, 0x51, 0x79, 0xf8, 0x31, 0x8a, 0x5e, 0x83, 0x8a, 0xcb, - 0x3f, 0x42, 0xbd, 0x21, 0xa6, 0x19, 0x79, 0x89, 0x7d, 0x52, 0x24, 0x58, 0x92, 0x95, 0xb9, 0xd1, - 0x5b, 0x50, 0xed, 0x3b, 0x76, 0x3f, 0xf4, 0x3c, 0x6c, 0xf7, 0xf7, 0x36, 0x8c, 0x4d, 0x4c, 0xb3, - 0xef, 0x12, 0xdb, 0x2a, 0x09, 0x92, 0xbc, 0x55, 0x12, 0x24, 0xf4, 0x22, 0x94, 0xa3, 0xee, 0x05, - 0x4d, 0xb0, 0xcb, 0xbc, 0x10, 0x16, 0xa0, 0x24, 0x1c, 0x73, 0x92, 0xc1, 0x5b, 0x7e, 0x94, 0xa5, - 0xd1, 0xa4, 0x99, 0x0f, 0x5e, 0x82, 0xe5, 0xc1, 0x4b, 0xb0, 0xfe, 0x7b, 0x0d, 0x16, 0xb3, 0xd6, - 0x3d, 0xb1, 0x07, 0xb5, 0x7b, 0xb2, 0x07, 0xdf, 0x85, 0x92, 0xeb, 0x98, 0x5d, 0xdf, 0xc5, 0x7d, - 0x1e, 0x66, 0x12, 0x3b, 0xf0, 0xb2, 0x63, 0x6e, 0xb8, 0xb8, 0xff, 0x4d, 0x2b, 0xd8, 0x5a, 0xdd, - 0x71, 0x2c, 0xf3, 0x82, 0xe5, 0xf3, 0xad, 0xe2, 0x32, 0x8a, 0xf2, 0x59, 0x2f, 0x72, 0xb0, 0x5d, - 0x82, 0x02, 0xb3, 0xa2, 0xff, 0x21, 0x0f, 0xb5, 0xe4, 0x5e, 0xfb, 0x5f, 0x7a, 0x15, 0x74, 0x0d, - 0x8a, 0x16, 0xcb, 0x73, 0xf9, 0x67, 0xff, 0xff, 0xa4, 0x40, 0xdc, 0x8c, 0xbb, 0x74, 0xcd, 0x9d, - 0xe7, 0x9a, 0x3c, 0x21, 0xa6, 0x53, 0x40, 0x35, 0x73, 0x49, 0x55, 0x33, 0x07, 0x51, 0x07, 0x8a, - 0x3e, 0xf6, 0x76, 0xac, 0x3e, 0xe6, 0x11, 0xa5, 0x21, 0x6b, 0xee, 0x3b, 0x1e, 0x26, 0x3a, 0x37, - 0x18, 0x4b, 0xac, 0x93, 0xcb, 0xa8, 0x3a, 0x39, 0x88, 0xde, 0x85, 0x72, 0xdf, 0xb1, 0x37, 0xad, - 0xc1, 0xba, 0xe1, 0xf2, 0x98, 0x72, 0x2c, 0x4b, 0xeb, 0x69, 0xc1, 0xc4, 0x3b, 0x07, 0xe2, 0x31, - 0xd1, 0x39, 0x88, 0xb8, 0xe2, 0x05, 0xfd, 0xc7, 0x0c, 0x40, 0xbc, 0x38, 0xe8, 0x15, 0xa8, 0xe0, - 0x5d, 0xdc, 0x0f, 0x03, 0xc7, 0x13, 0xc1, 0x9d, 0x37, 0xe2, 0x04, 0xac, 0x44, 0x63, 0x88, 0x51, - 0xe2, 0x5d, 0xb6, 0x31, 0xc2, 0xbe, 0x6b, 0xf4, 0x45, 0x07, 0x8f, 0x0e, 0x26, 0x02, 0x65, 0xef, - 0x8a, 0x40, 0xf4, 0xff, 0x30, 0x43, 0x7b, 0x7e, 0xac, 0x79, 0x87, 0x26, 0xe3, 0xc6, 0x82, 0xad, - 0x76, 0xfb, 0x28, 0x1d, 0xbd, 0x01, 0xf3, 0xdb, 0xd1, 0xc6, 0x23, 0x63, 0x9b, 0xa1, 0x02, 0x34, - 0x1f, 0x8b, 0x09, 0xca, 0xe8, 0xe6, 0x64, 0x1c, 0x6d, 0x42, 0xc5, 0xb0, 0x6d, 0x27, 0xa0, 0x1f, - 0x0e, 0xd1, 0xd0, 0x7b, 0x72, 0xda, 0x36, 0x6d, 0xae, 0xc6, 0xbc, 0x2c, 0xb5, 0xa1, 0x1e, 0x2f, - 0x69, 0x90, 0x3d, 0x5e, 0x82, 0x51, 0x07, 0x0a, 0x43, 0xa3, 0x87, 0x87, 0x22, 0x52, 0x3f, 0x3e, - 0xd5, 0xc4, 0x05, 0xca, 0xc6, 0xb4, 0xd3, 0xef, 0x34, 0x93, 0x93, 0xbf, 0xd3, 0x0c, 0x59, 0xde, - 0x84, 0x5a, 0x72, 0x3c, 0xfb, 0xcb, 0x3a, 0x9e, 0x94, 0xb3, 0x8e, 0xf2, 0x6d, 0xf3, 0x1c, 0x03, - 0x2a, 0xd2, 0xa0, 0xee, 0x87, 0x09, 0xfd, 0x97, 0x1a, 0x2c, 0x66, 0xf9, 0x2e, 0x5a, 0x97, 0x3c, - 0x5e, 0xe3, 0x8d, 0x89, 0x8c, 0xad, 0xce, 0x65, 0xa7, 0xb8, 0x7a, 0xec, 0xe8, 0x6d, 0x58, 0xb0, - 0x1d, 0x13, 0x77, 0x0d, 0x62, 0x60, 0x68, 0xf9, 0x41, 0x3d, 0x47, 0x1b, 0xbe, 0xb4, 0xa1, 0x41, - 0x28, 0xab, 0x82, 0x20, 0x49, 0xcf, 0x2b, 0x04, 0xfd, 0x07, 0x1a, 0x54, 0x13, 0xfd, 0xc6, 0xbb, - 0xce, 0x7c, 0xe4, 0x7c, 0x25, 0xb7, 0xbf, 0x7c, 0x45, 0xff, 0x49, 0x0e, 0x2a, 0x52, 0x31, 0x76, - 0xd7, 0x63, 0xb8, 0x0e, 0x55, 0xfe, 0x79, 0xb3, 0xec, 0x01, 0xab, 0x81, 0x72, 0xbc, 0xb3, 0x90, - 0x6a, 0xef, 0x9f, 0x77, 0x7a, 0x1b, 0x11, 0x2f, 0x2d, 0x81, 0x68, 0xdb, 0xc9, 0x57, 0x30, 0xc9, - 0xc4, 0x82, 0x4a, 0x41, 0xd7, 0x60, 0x29, 0x74, 0x4d, 0x23, 0xc0, 0x5d, 0x9f, 0x37, 0xca, 0xbb, - 0x76, 0x38, 0xea, 0x61, 0x8f, 0x7a, 0xfc, 0x2c, 0x6b, 0x94, 0x30, 0x0e, 0xd1, 0x49, 0xbf, 0x48, - 0xe9, 0x92, 0xce, 0xc5, 0x2c, 0xba, 0x7e, 0x16, 0x50, 0xba, 0x19, 0xac, 0xcc, 0xaf, 0xb6, 0xcf, - 0xf9, 0xfd, 0x58, 0x83, 0x5a, 0xb2, 0xc7, 0xfb, 0x40, 0x16, 0x7a, 0x0f, 0xca, 0x51, 0xbf, 0xf6, - 0xae, 0x07, 0xf0, 0x34, 0x14, 0x3c, 0x6c, 0xf8, 0x8e, 0xcd, 0x3d, 0x93, 0x86, 0x18, 0x86, 0xc8, - 0x21, 0x86, 0x21, 0xfa, 0x15, 0x98, 0x63, 0x33, 0xf8, 0xa6, 0x35, 0x0c, 0xb0, 0x87, 0xce, 0x40, - 0xc1, 0x0f, 0x8c, 0x00, 0xfb, 0x75, 0xed, 0x78, 0xfe, 0xe4, 0xc2, 0xa9, 0xa5, 0x74, 0x6b, 0x96, - 0x90, 0x99, 0x56, 0xc6, 0x29, 0x6b, 0x65, 0x88, 0xfe, 0x3d, 0x0d, 0xe6, 0xe4, 0x0e, 0xf4, 0xbd, - 0x51, 0xfb, 0x15, 0x5f, 0xed, 0x23, 0x31, 0x86, 0xe1, 0xbd, 0x59, 0xd9, 0xaf, 0x66, 0xfd, 0x37, - 0x1a, 0x9b, 0xd9, 0xa8, 0x75, 0x79, 0xb7, 0xe6, 0x07, 0x71, 0xff, 0x82, 0x78, 0x98, 0x4f, 0x03, - 0xdb, 0x7e, 0xfb, 0x17, 0x34, 0xfc, 0x29, 0xe2, 0x72, 0xf8, 0x53, 0x08, 0xfa, 0x9f, 0x72, 0x74, - 0xe4, 0x71, 0x9b, 0xfa, 0x41, 0x77, 0x6e, 0x12, 0xd9, 0x49, 0xfe, 0x2b, 0x64, 0x27, 0xcf, 0x40, - 0x91, 0x7e, 0x0e, 0xa2, 0xc4, 0x81, 0x2e, 0x1a, 0x81, 0xd4, 0x63, 0x42, 0x86, 0xdc, 0x22, 0x6a, - 0xcd, 0xde, 0x65, 0xd4, 0xfa, 0x97, 0x06, 0x0b, 0x6a, 0x1f, 0xff, 0x81, 0x4f, 0x6b, 0x6a, 0x43, - 0xe5, 0xef, 0xd3, 0x86, 0xfa, 0xa7, 0x06, 0xf3, 0xca, 0xf1, 0xc2, 0xc3, 0xf3, 0xea, 0x3f, 0xcb, - 0xc1, 0x52, 0xb6, 0x9a, 0xfb, 0x52, 0x3e, 0x9d, 0x05, 0x92, 0x08, 0x9d, 0x8b, 0xbf, 0xec, 0x87, - 0x53, 0xd5, 0x13, 0x7d, 0x05, 0x91, 0x45, 0xa5, 0xce, 0x05, 0x84, 0x38, 0xba, 0x06, 0x15, 0x4b, - 0x3a, 0x81, 0xc8, 0x67, 0x35, 0x8a, 0xe5, 0x73, 0x07, 0x56, 0x18, 0x4f, 0x39, 0x6d, 0x90, 0x55, - 0xb5, 0x0b, 0x30, 0x43, 0x52, 0x0f, 0x7d, 0x07, 0x8a, 0x7c, 0x38, 0xe8, 0x79, 0x28, 0x53, 0x2f, - 0xa5, 0x15, 0x01, 0x4b, 0x3b, 0xe9, 0x47, 0x93, 0x80, 0x89, 0x3b, 0x00, 0x25, 0x81, 0xa1, 0x97, - 0x00, 0x48, 0xe2, 0xc8, 0xfd, 0x33, 0x47, 0xfd, 0x93, 0x56, 0x1e, 0xae, 0x63, 0xa6, 0x9c, 0xb2, - 0x1c, 0x81, 0xfa, 0xaf, 0x72, 0x50, 0x91, 0xcf, 0x3c, 0xee, 0xc8, 0xf8, 0x47, 0x20, 0xaa, 0xc2, - 0xae, 0x61, 0x9a, 0xe4, 0x2f, 0x16, 0x01, 0xb9, 0x35, 0x75, 0x92, 0xc4, 0xff, 0xab, 0x42, 0x82, - 0xd5, 0x00, 0xf4, 0x54, 0xd9, 0x4a, 0x90, 0x24, 0xab, 0xb5, 0x24, 0x6d, 0x79, 0x1b, 0x0e, 0x67, - 0xaa, 0x92, 0x33, 0xf7, 0xd9, 0x7b, 0x95, 0xb9, 0xff, 0x76, 0x16, 0x0e, 0x67, 0x9e, 0x35, 0x3d, - 0x70, 0x2f, 0x56, 0x3d, 0x28, 0x7f, 0x4f, 0x3c, 0xe8, 0x63, 0x2d, 0x6b, 0x65, 0x59, 0xdf, 0xfe, - 0x95, 0x7d, 0x1c, 0xc0, 0xdd, 0xab, 0x35, 0x56, 0xb7, 0xe5, 0xec, 0x1d, 0xf9, 0x44, 0x61, 0xbf, - 0x3e, 0x81, 0x9e, 0x65, 0x45, 0x18, 0xb5, 0x55, 0xa4, 0xb6, 0x44, 0x84, 0x48, 0x98, 0x2a, 0x72, - 0x88, 0xd4, 0xe5, 0x42, 0x82, 0x95, 0xfe, 0xa5, 0xb8, 0x2e, 0xe7, 0x3c, 0xc9, 0xea, 0x7f, 0x4e, - 0xc6, 0xff, 0xbb, 0x7b, 0xf8, 0xdf, 0x1a, 0x54, 0x13, 0x87, 0xcf, 0x0f, 0xcf, 0x37, 0xe8, 0x47, - 0x1a, 0x94, 0xa3, 0x7b, 0x0f, 0x77, 0x9d, 0x86, 0xae, 0x42, 0x01, 0xb3, 0xb3, 0x77, 0x16, 0xee, - 0x0e, 0x25, 0xee, 0x46, 0x11, 0x1a, 0xbf, 0x0d, 0x95, 0x38, 0x6e, 0xef, 0x70, 0x41, 0xfd, 0x8f, - 0x9a, 0x48, 0x30, 0xe3, 0x31, 0x3d, 0xd0, 0xa5, 0x88, 0xdf, 0x29, 0x7f, 0xa7, 0xef, 0xf4, 0xbb, - 0x12, 0xcc, 0x52, 0x3e, 0x52, 0x00, 0x06, 0xd8, 0x1b, 0x59, 0xb6, 0x31, 0xa4, 0xaf, 0x53, 0x62, - 0x7e, 0x2b, 0x30, 0xd9, 0x6f, 0x05, 0x86, 0xb6, 0xa0, 0x1a, 0x37, 0xad, 0xa8, 0x9a, 0xec, 0x2b, - 0x57, 0x6f, 0xab, 0x4c, 0xac, 0x15, 0x9e, 0x90, 0x54, 0xcf, 0xa4, 0x13, 0x44, 0x64, 0xc2, 0x42, - 0xdf, 0xb1, 0x03, 0xc3, 0xb2, 0xb1, 0xc7, 0x0c, 0xe5, 0xb3, 0xae, 0x9c, 0x9c, 0x56, 0x78, 0x58, - 0xed, 0xaf, 0xca, 0xa9, 0x57, 0x4e, 0x54, 0x1a, 0xfa, 0x00, 0xe6, 0x45, 0x12, 0xce, 0x8c, 0xcc, - 0x64, 0x5d, 0x39, 0x59, 0x93, 0x59, 0xd8, 0x96, 0x56, 0xa4, 0xd4, 0x2b, 0x27, 0x0a, 0x09, 0x0d, - 0xa1, 0xe6, 0x3a, 0xe6, 0x55, 0x9b, 0xb7, 0x1d, 0x8c, 0xde, 0x10, 0xf3, 0x4e, 0xe9, 0x4a, 0x2a, - 0xe5, 0x51, 0xb8, 0x58, 0x28, 0x4e, 0xca, 0xaa, 0x97, 0xb8, 0x92, 0x54, 0xf4, 0x3e, 0xcc, 0x0d, - 0x49, 0x2d, 0xb4, 0xb6, 0xeb, 0x5a, 0x1e, 0x36, 0xb3, 0xaf, 0x5c, 0x5d, 0x90, 0x38, 0x58, 0x20, - 0x94, 0x65, 0xd4, 0x6b, 0x27, 0x32, 0x85, 0xac, 0xfe, 0xc8, 0xd8, 0xed, 0x84, 0xb6, 0xbf, 0xb6, - 0xcb, 0xaf, 0xcf, 0x14, 0xb3, 0x56, 0x7f, 0x5d, 0x65, 0x62, 0xab, 0x9f, 0x90, 0x54, 0x57, 0x3f, - 0x41, 0x44, 0x17, 0x68, 0x9c, 0x67, 0x4b, 0xc2, 0xae, 0x5e, 0x2d, 0xa5, 0x66, 0x8b, 0xad, 0x06, - 0x6b, 0x5a, 0xf0, 0x27, 0x45, 0x69, 0xa4, 0x81, 0xaf, 0x01, 0x7d, 0xed, 0x0e, 0x0e, 0x42, 0xcf, - 0xc6, 0x26, 0xbf, 0x75, 0x95, 0x5e, 0x03, 0x85, 0x2b, 0x5a, 0x03, 0x05, 0x4d, 0xad, 0x81, 0x42, - 0x25, 0x7b, 0xca, 0x75, 0xcc, 0x2b, 0xcc, 0x65, 0x82, 0xe8, 0x2e, 0xd6, 0xa3, 0x29, 0x53, 0x31, - 0x0b, 0xdb, 0x53, 0x8a, 0x94, 0xba, 0xa7, 0x14, 0x12, 0xbf, 0xfe, 0x23, 0x5f, 0x16, 0x61, 0x33, - 0x55, 0x99, 0x72, 0xfd, 0x27, 0xc5, 0x19, 0x5d, 0xff, 0x49, 0x51, 0x52, 0xd7, 0x7f, 0xd2, 0xb2, - 0x25, 0xd1, 0x5e, 0xd0, 0x3f, 0xd1, 0xa0, 0x9a, 0xf0, 0x74, 0xf4, 0x3a, 0x44, 0xd7, 0x0c, 0xae, - 0xec, 0xb9, 0x22, 0x51, 0x55, 0xae, 0x25, 0x10, 0x3c, 0xeb, 0x5a, 0x02, 0xc1, 0xd1, 0x05, 0x80, - 0xe8, 0xab, 0x70, 0xab, 0x30, 0x49, 0xb3, 0xa4, 0x98, 0x53, 0xce, 0x92, 0x62, 0x54, 0xff, 0x3c, - 0x0f, 0x25, 0xb1, 0x55, 0xee, 0x4b, 0x21, 0xd3, 0x82, 0xe2, 0x08, 0xfb, 0xf4, 0x7a, 0x42, 0x2e, - 0xce, 0x47, 0x38, 0x24, 0xe7, 0x23, 0x1c, 0x52, 0xd3, 0xa5, 0xfc, 0x1d, 0xa5, 0x4b, 0x33, 0xfb, - 0x4e, 0x97, 0x30, 0x3d, 0x9a, 0x94, 0x02, 0x9e, 0x38, 0x57, 0xb8, 0x75, 0x14, 0x15, 0x07, 0x97, - 0xb2, 0x60, 0xe2, 0xe0, 0x52, 0x26, 0xa1, 0x6d, 0x38, 0x28, 0x9d, 0x7d, 0xf0, 0xde, 0x13, 0x09, - 0x3d, 0x0b, 0xd3, 0xcf, 0x81, 0x3b, 0x94, 0x8b, 0x39, 0xd8, 0x76, 0x02, 0x95, 0xf3, 0xcd, 0x24, - 0x4d, 0xff, 0x5b, 0x0e, 0x16, 0xd4, 0xf1, 0xde, 0x97, 0x85, 0x7d, 0x1e, 0xca, 0x78, 0xd7, 0x0a, - 0xba, 0x7d, 0xc7, 0xc4, 0xbc, 0x68, 0xa3, 0xeb, 0x44, 0xc0, 0xd3, 0x8e, 0xa9, 0xac, 0x93, 0xc0, - 0xe4, 0xdd, 0x90, 0xdf, 0xd7, 0x6e, 0x88, 0x5b, 0x75, 0x33, 0xb7, 0x6f, 0xd5, 0x65, 0xcf, 0x73, - 0xf9, 0x3e, 0xcd, 0xf3, 0xcd, 0x1c, 0xd4, 0x92, 0xf1, 0xf0, 0xeb, 0xe1, 0x42, 0xaa, 0x37, 0xe4, - 0xf7, 0xed, 0x0d, 0x6f, 0xc0, 0x3c, 0xc9, 0xde, 0x8c, 0x20, 0xe0, 0x17, 0xf7, 0x66, 0x68, 0xd6, - 0xc3, 0x62, 0x53, 0x68, 0xaf, 0x0a, 0x5c, 0x89, 0x4d, 0x12, 0xae, 0x7f, 0x37, 0x07, 0xf3, 0x4a, - 0xdc, 0x7e, 0xf8, 0x42, 0x8a, 0x5e, 0x85, 0x79, 0x25, 0x1d, 0xd2, 0xbf, 0xcf, 0xf6, 0x89, 0x9a, - 0x87, 0x3c, 0x7c, 0xf3, 0xb2, 0x00, 0x73, 0x72, 0x5e, 0xa5, 0xb7, 0xa1, 0x9a, 0x48, 0x83, 0xe4, - 0x17, 0xd0, 0xf6, 0xf3, 0x02, 0xfa, 0x12, 0x2c, 0x66, 0x7d, 0xbd, 0xf5, 0x4f, 0x35, 0x4a, 0x48, - 0xdf, 0xcc, 0x3d, 0x0b, 0x60, 0xe3, 0x1b, 0xdd, 0xdb, 0xd6, 0x4d, 0x6c, 0x1a, 0xf0, 0x8d, 0xf3, - 0x89, 0x32, 0xa3, 0x24, 0x30, 0xa2, 0xc9, 0x19, 0x9a, 0xdd, 0xdb, 0x56, 0x2b, 0x54, 0x93, 0x33, - 0x34, 0x53, 0x9a, 0x04, 0xa6, 0xff, 0x30, 0x2f, 0x4a, 0xda, 0xf8, 0x6a, 0xeb, 0x7b, 0x50, 0x73, - 0xc5, 0xc3, 0xed, 0x47, 0x4b, 0x93, 0xfa, 0x88, 0x3f, 0x69, 0x69, 0x41, 0xa5, 0xa8, 0xba, 0x79, - 0xb5, 0x96, 0xdb, 0xa7, 0xee, 0x4e, 0xa2, 0x6c, 0x5b, 0x50, 0x29, 0xe8, 0xdb, 0x70, 0x50, 0xdc, - 0xfc, 0xd9, 0xc1, 0x62, 0xe0, 0xf9, 0xa9, 0xca, 0xd9, 0x4d, 0xdc, 0x48, 0x20, 0x39, 0xf2, 0x6a, - 0x82, 0x94, 0x50, 0xcf, 0xc7, 0x3e, 0xb3, 0x5f, 0xf5, 0xc9, 0xc1, 0x57, 0x13, 0x24, 0x52, 0x5f, - 0x57, 0x13, 0x97, 0x85, 0xd1, 0x19, 0x28, 0xd1, 0xdf, 0x12, 0xdd, 0x7a, 0x05, 0xe8, 0x46, 0xa5, - 0x7c, 0x8a, 0x85, 0x22, 0x87, 0xd0, 0x8b, 0x50, 0x8e, 0xee, 0x14, 0xf3, 0xc3, 0x44, 0xe6, 0x33, - 0x02, 0x54, 0x7c, 0x46, 0x80, 0xfa, 0xcf, 0x35, 0x38, 0x32, 0xf5, 0x22, 0xf1, 0x83, 0x2e, 0xb6, - 0x9f, 0x7a, 0x16, 0x4a, 0xe2, 0xb8, 0x0f, 0x01, 0x14, 0xde, 0xb9, 0xba, 0x76, 0x75, 0xed, 0x4c, - 0xed, 0x00, 0xaa, 0x40, 0xf1, 0xf2, 0xda, 0xc5, 0x33, 0xe7, 0x2e, 0xbe, 0x55, 0xd3, 0xc8, 0x43, - 0xe7, 0xea, 0xc5, 0x8b, 0xe4, 0x21, 0xf7, 0xd4, 0x05, 0xf9, 0xf2, 0x11, 0xfb, 0x8c, 0xa2, 0x39, - 0x28, 0xad, 0xba, 0x2e, 0xf5, 0x5b, 0x26, 0xbb, 0xb6, 0x63, 0x11, 0x5f, 0xad, 0x69, 0xa8, 0x08, - 0xf9, 0x4b, 0x97, 0xd6, 0x6b, 0x39, 0xb4, 0x08, 0xb5, 0x33, 0xd8, 0x30, 0x87, 0x96, 0x8d, 0x45, - 0xb0, 0xa8, 0xe5, 0xdb, 0xd7, 0x3f, 0xfb, 0x62, 0x45, 0xfb, 0xfc, 0x8b, 0x15, 0xed, 0xaf, 0x5f, - 0xac, 0x68, 0x37, 0xbf, 0x5c, 0x39, 0xf0, 0xf9, 0x97, 0x2b, 0x07, 0xfe, 0xfc, 0xe5, 0xca, 0x81, - 0xf7, 0x9e, 0x95, 0x7e, 0x37, 0xc7, 0xde, 0xc9, 0xf5, 0x1c, 0x12, 0x27, 0xf9, 0x53, 0x2b, 0xf9, - 0x4b, 0xc1, 0x4f, 0x73, 0xc7, 0x56, 0xe9, 0xe3, 0x65, 0xc6, 0xd7, 0x3c, 0xe7, 0x34, 0x19, 0x40, - 0x7f, 0xec, 0xe5, 0xf7, 0x0a, 0xf4, 0x47, 0x5d, 0xcf, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, 0xb4, - 0x0d, 0xed, 0xa1, 0x64, 0x38, 0x00, 0x00, + // 3467 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5b, 0x4b, 0x6c, 0x1b, 0xd7, + 0xd5, 0xf6, 0x90, 0x12, 0x1f, 0x87, 0x92, 0x48, 0x5f, 0xcb, 0x0a, 0xad, 0xd8, 0xa2, 0x33, 0xce, + 0xff, 0xc7, 0x09, 0x12, 0x32, 0x71, 0x1e, 0xc8, 0xe3, 0x47, 0x02, 0xd1, 0x56, 0xfc, 0x88, 0x65, + 0x3b, 0x94, 0x95, 0xdf, 0x0d, 0x52, 0x30, 0x43, 0xce, 0x15, 0x35, 0x12, 0x39, 0x33, 0x99, 0x87, + 0x2c, 0x01, 0x59, 0xb4, 0x45, 0x9b, 0xee, 0x52, 0x03, 0xed, 0xa2, 0x40, 0x17, 0xe9, 0xb6, 0x01, + 0xba, 0x2b, 0xd0, 0x75, 0x57, 0xcd, 0xa2, 0x28, 0xd2, 0xae, 0xba, 0x62, 0x8b, 0x04, 0x5d, 0x94, + 0x8b, 0xae, 0xdb, 0xae, 0x8a, 0xfb, 0x9a, 0xb9, 0x77, 0x38, 0xb4, 0xe4, 0x57, 0x9d, 0xc2, 0x2b, + 0x69, 0xbe, 0xf3, 0xba, 0x73, 0xcf, 0xbd, 0x67, 0xce, 0x39, 0xf7, 0x12, 0x4e, 0xb8, 0xdb, 0xbd, + 0x86, 0xe1, 0x0d, 0x0c, 0xd3, 0xc0, 0x3b, 0xd8, 0x0e, 0xfc, 0x06, 0xfb, 0x53, 0x77, 0x3d, 0x27, + 0x70, 0xd0, 0x8c, 0x4c, 0x5a, 0xd4, 0xb7, 0x5f, 0xf5, 0xeb, 0x96, 0xd3, 0x30, 0x5c, 0xab, 0xd1, + 0x75, 0x3c, 0xdc, 0xd8, 0x79, 0xa1, 0xd1, 0xc3, 0x36, 0xf6, 0x8c, 0x00, 0x9b, 0x4c, 0x62, 0xf1, + 0xb4, 0xc4, 0x63, 0xe3, 0xe0, 0xa6, 0xe3, 0x6d, 0x5b, 0x76, 0x2f, 0x8d, 0xb3, 0xd6, 0x73, 0x9c, + 0x5e, 0x1f, 0x37, 0xe8, 0x53, 0x27, 0xdc, 0x68, 0x04, 0xd6, 0x00, 0xfb, 0x81, 0x31, 0x70, 0x39, + 0xc3, 0x4b, 0xb1, 0xaa, 0x81, 0xd1, 0xdd, 0xb4, 0x6c, 0xec, 0xed, 0x35, 0xe8, 0x78, 0x5d, 0xab, + 0xe1, 0x61, 0xdf, 0x09, 0xbd, 0x2e, 0x1e, 0x53, 0xfb, 0x5c, 0xcf, 0x0a, 0x36, 0xc3, 0x4e, 0xbd, + 0xeb, 0x0c, 0x1a, 0x3d, 0xa7, 0xe7, 0xc4, 0xfa, 0xc9, 0x13, 0x7d, 0xa0, 0xff, 0x71, 0xf6, 0xd7, + 0x2d, 0x3b, 0xc0, 0x9e, 0x6d, 0xf4, 0x1b, 0x7e, 0x77, 0x13, 0x9b, 0x61, 0x1f, 0x7b, 0xf1, 0x7f, + 0x4e, 0x67, 0x0b, 0x77, 0x03, 0x7f, 0x0c, 0x60, 0xb2, 0xfa, 0xad, 0x79, 0x98, 0x5d, 0x21, 0x53, + 0xb3, 0x86, 0x3f, 0x0a, 0xb1, 0xdd, 0xc5, 0xe8, 0x69, 0x98, 0xfe, 0x28, 0xc4, 0x21, 0xae, 0x6a, + 0x27, 0xb5, 0xd3, 0xc5, 0xe6, 0x91, 0xd1, 0xb0, 0x56, 0xa6, 0xc0, 0xb3, 0xce, 0xc0, 0x0a, 0xf0, + 0xc0, 0x0d, 0xf6, 0x5a, 0x8c, 0x03, 0xbd, 0x0e, 0x33, 0x5b, 0x4e, 0xa7, 0xed, 0xe3, 0xa0, 0x6d, + 0x1b, 0x03, 0x5c, 0xcd, 0x50, 0x89, 0xea, 0x68, 0x58, 0x9b, 0xdf, 0x72, 0x3a, 0x6b, 0x38, 0xb8, + 0x62, 0x0c, 0x64, 0x31, 0x88, 0x51, 0xf4, 0x1c, 0xe4, 0x43, 0x1f, 0x7b, 0x6d, 0xcb, 0xac, 0x66, + 0xa9, 0xd8, 0xfc, 0x68, 0x58, 0xab, 0x10, 0xe8, 0xa2, 0x29, 0x89, 0xe4, 0x18, 0x82, 0x9e, 0x85, + 0x5c, 0xcf, 0x73, 0x42, 0xd7, 0xaf, 0x4e, 0x9d, 0xcc, 0x0a, 0x6e, 0x86, 0xc8, 0xdc, 0x0c, 0x41, + 0x57, 0x21, 0xc7, 0xfc, 0x5d, 0x9d, 0x3e, 0x99, 0x3d, 0x5d, 0x3a, 0xf3, 0x44, 0x5d, 0x5e, 0x04, + 0x75, 0xe5, 0x85, 0xd9, 0x13, 0x53, 0xc8, 0xe8, 0xb2, 0x42, 0xbe, 0x6c, 0xfe, 0x76, 0x18, 0xa6, + 0x29, 0x1f, 0xba, 0x0a, 0xf9, 0xae, 0x87, 0x89, 0xb3, 0xaa, 0xe8, 0xa4, 0x76, 0xba, 0x74, 0x66, + 0xb1, 0xce, 0x16, 0x41, 0x5d, 0x38, 0xa9, 0x7e, 0x5d, 0x2c, 0x82, 0xe6, 0xb1, 0xd1, 0xb0, 0x76, + 0x98, 0xb3, 0xc7, 0x5a, 0x6f, 0xfd, 0xb9, 0xa6, 0xb5, 0x84, 0x16, 0x74, 0x0d, 0x8a, 0x7e, 0xd8, + 0x19, 0x58, 0xc1, 0x25, 0xa7, 0x43, 0xe7, 0xbc, 0x74, 0xe6, 0x31, 0x75, 0xb8, 0x6b, 0x82, 0xdc, + 0x7c, 0x6c, 0x34, 0xac, 0x1d, 0x89, 0xb8, 0x63, 0x8d, 0x17, 0x0e, 0xb5, 0x62, 0x25, 0x68, 0x13, + 0xca, 0x1e, 0x76, 0x3d, 0xcb, 0xf1, 0xac, 0xc0, 0xf2, 0x31, 0xd1, 0x9b, 0xa1, 0x7a, 0x4f, 0xa8, + 0x7a, 0x5b, 0x2a, 0x53, 0xf3, 0xc4, 0x68, 0x58, 0x3b, 0x96, 0x90, 0x54, 0x6c, 0x24, 0xd5, 0xa2, + 0x00, 0x50, 0x02, 0x5a, 0xc3, 0x01, 0xf5, 0x67, 0xe9, 0xcc, 0xc9, 0xdb, 0x1a, 0x5b, 0xc3, 0x41, + 0xf3, 0xe4, 0x68, 0x58, 0x3b, 0x3e, 0x2e, 0xaf, 0x98, 0x4c, 0xd1, 0x8f, 0xfa, 0x50, 0x91, 0x51, + 0x93, 0xbc, 0xe0, 0x14, 0xb5, 0xb9, 0x34, 0xd9, 0x26, 0xe1, 0x6a, 0x2e, 0x8d, 0x86, 0xb5, 0xc5, + 0xa4, 0xac, 0x62, 0x6f, 0x4c, 0x33, 0xf1, 0x4f, 0xd7, 0xb0, 0xbb, 0xb8, 0x4f, 0xcc, 0x4c, 0xa7, + 0xf9, 0xe7, 0xac, 0x20, 0x33, 0xff, 0x44, 0xdc, 0xaa, 0x7f, 0x22, 0x18, 0x7d, 0x00, 0x33, 0xd1, + 0x03, 0x99, 0xaf, 0x1c, 0x5f, 0x47, 0xe9, 0x4a, 0xc9, 0x4c, 0x2d, 0x8e, 0x86, 0xb5, 0x05, 0x59, + 0x46, 0x51, 0xad, 0x68, 0x8b, 0xb5, 0xf7, 0xd9, 0xcc, 0xe4, 0x27, 0x6b, 0x67, 0x1c, 0xb2, 0xf6, + 0xfe, 0xf8, 0x8c, 0x28, 0xda, 0x88, 0x76, 0xb2, 0x89, 0xc3, 0x6e, 0x17, 0x63, 0x13, 0x9b, 0xd5, + 0x42, 0x9a, 0xf6, 0x4b, 0x12, 0x07, 0xd3, 0x2e, 0xcb, 0xa8, 0xda, 0x65, 0x0a, 0x99, 0xeb, 0x2d, + 0xa7, 0xb3, 0xe2, 0x79, 0x8e, 0xe7, 0x57, 0x8b, 0x69, 0x73, 0x7d, 0x49, 0x90, 0xd9, 0x5c, 0x47, + 0xdc, 0xea, 0x5c, 0x47, 0x30, 0x1f, 0x6f, 0x2b, 0xb4, 0x2f, 0x63, 0xc3, 0xc7, 0x66, 0x15, 0x26, + 0x8c, 0x37, 0xe2, 0x88, 0xc6, 0x1b, 0x21, 0x63, 0xe3, 0x8d, 0x28, 0xc8, 0x84, 0x39, 0xf6, 0xbc, + 0xec, 0xfb, 0x56, 0xcf, 0xc6, 0x66, 0xb5, 0x44, 0xf5, 0x1f, 0x4f, 0xd3, 0x2f, 0x78, 0x9a, 0xc7, + 0x47, 0xc3, 0x5a, 0x55, 0x95, 0x53, 0x6c, 0x24, 0x74, 0xa2, 0x0f, 0x61, 0x96, 0x21, 0xad, 0xd0, + 0xb6, 0x2d, 0xbb, 0x57, 0x9d, 0xa1, 0x46, 0x1e, 0x4f, 0x33, 0xc2, 0x59, 0x9a, 0x8f, 0x8f, 0x86, + 0xb5, 0xc7, 0x14, 0x29, 0xc5, 0x84, 0xaa, 0x90, 0x44, 0x0c, 0x06, 0xc4, 0x8e, 0x9d, 0x4d, 0x8b, + 0x18, 0x97, 0x54, 0x26, 0x16, 0x31, 0x12, 0x92, 0x6a, 0xc4, 0x48, 0x10, 0x63, 0x7f, 0x70, 0x27, + 0xcf, 0x4d, 0xf6, 0x07, 0xf7, 0xb3, 0xe4, 0x8f, 0x14, 0x57, 0x2b, 0xda, 0xd0, 0xc7, 0x40, 0x3e, + 0x3c, 0xe7, 0x42, 0xb7, 0x6f, 0x75, 0x8d, 0x00, 0x9f, 0xc3, 0x01, 0xee, 0x92, 0x48, 0x5d, 0xa6, + 0x56, 0xf4, 0x31, 0x2b, 0x63, 0x9c, 0x4d, 0x7d, 0x34, 0xac, 0x2d, 0xa5, 0xe9, 0x50, 0xac, 0xa6, + 0x5a, 0x41, 0xdf, 0xd1, 0xe0, 0xa8, 0x1f, 0x18, 0xb6, 0x69, 0xf4, 0x1d, 0x1b, 0x5f, 0xb4, 0x7b, + 0x1e, 0xf6, 0xfd, 0x8b, 0xf6, 0x86, 0x53, 0xad, 0x50, 0xfb, 0xa7, 0x12, 0x61, 0x3d, 0x8d, 0xb5, + 0x79, 0x6a, 0x34, 0xac, 0xd5, 0x52, 0xb5, 0x28, 0x23, 0x48, 0x37, 0x84, 0x76, 0xe1, 0x88, 0xc8, + 0x2a, 0xd6, 0x03, 0xab, 0x6f, 0xf9, 0x46, 0x60, 0x39, 0x76, 0xf5, 0x30, 0xb5, 0xff, 0x44, 0x32, + 0x3a, 0x8e, 0x31, 0x36, 0x9f, 0x18, 0x0d, 0x6b, 0x27, 0x52, 0x34, 0x28, 0xb6, 0xd3, 0x4c, 0xc4, + 0x4b, 0xe8, 0x9a, 0x87, 0x09, 0x23, 0x36, 0xab, 0x47, 0x26, 0x2f, 0xa1, 0x88, 0x49, 0x5e, 0x42, + 0x11, 0x98, 0xb6, 0x84, 0x22, 0x22, 0xb1, 0xe4, 0x1a, 0x5e, 0x60, 0x11, 0xb3, 0xab, 0x86, 0xb7, + 0x8d, 0xbd, 0xea, 0x7c, 0x9a, 0xa5, 0x6b, 0x2a, 0x13, 0xb3, 0x94, 0x90, 0x54, 0x2d, 0x25, 0x88, + 0xe8, 0x96, 0x06, 0xea, 0xd0, 0x2c, 0xc7, 0x6e, 0x91, 0xb4, 0xc1, 0x27, 0xaf, 0x77, 0x94, 0x1a, + 0x7d, 0xea, 0x36, 0xaf, 0x27, 0xb3, 0x37, 0x9f, 0x1a, 0x0d, 0x6b, 0xa7, 0x26, 0x6a, 0x53, 0x06, + 0x32, 0xd9, 0x28, 0xba, 0x01, 0x25, 0x42, 0xc4, 0x34, 0x01, 0x33, 0xab, 0x0b, 0x74, 0x0c, 0xc7, + 0xc6, 0xc7, 0xc0, 0x19, 0x68, 0x06, 0x72, 0x54, 0x92, 0x50, 0xec, 0xc8, 0xaa, 0x9a, 0x79, 0x98, + 0xa6, 0xf2, 0xfa, 0x28, 0x07, 0x47, 0x52, 0xd6, 0x06, 0x7a, 0x13, 0x72, 0x5e, 0x68, 0x93, 0x84, + 0x8d, 0x65, 0x29, 0x48, 0xb5, 0xba, 0x1e, 0x5a, 0x26, 0xcb, 0x16, 0xbd, 0xd0, 0x56, 0x72, 0xb8, + 0x69, 0x0a, 0x10, 0x79, 0x92, 0x2d, 0x5a, 0x26, 0xcf, 0x46, 0x26, 0xca, 0x6f, 0x39, 0x1d, 0x55, + 0x9e, 0x02, 0x08, 0xc3, 0xac, 0x58, 0x78, 0x6d, 0x8b, 0xec, 0x2a, 0x96, 0x67, 0x3c, 0xa9, 0xaa, + 0x79, 0x27, 0xec, 0x60, 0xcf, 0xc6, 0x01, 0xf6, 0xc5, 0x3b, 0xd0, 0x6d, 0x45, 0xa3, 0x88, 0x27, + 0x21, 0x92, 0xfe, 0x19, 0x19, 0x47, 0x3f, 0xd1, 0xa0, 0x3a, 0x30, 0x76, 0xdb, 0x02, 0xf4, 0xdb, + 0x1b, 0x8e, 0xd7, 0x76, 0xb1, 0x67, 0x39, 0x26, 0x4d, 0x3e, 0x4b, 0x67, 0xfe, 0x6f, 0xdf, 0x8d, + 0x54, 0x5f, 0x35, 0x76, 0x05, 0xec, 0xbf, 0xed, 0x78, 0xd7, 0xa8, 0xf8, 0x8a, 0x1d, 0x78, 0x7b, + 0xcd, 0x13, 0x5f, 0x0c, 0x6b, 0x87, 0x88, 0x5b, 0x06, 0x69, 0x3c, 0xad, 0x74, 0x18, 0xfd, 0x48, + 0x83, 0x85, 0xc0, 0x09, 0x8c, 0x7e, 0xbb, 0x1b, 0x0e, 0xc2, 0xbe, 0x11, 0x58, 0x3b, 0xb8, 0x1d, + 0xfa, 0x46, 0x0f, 0xf3, 0x1c, 0xf7, 0x8d, 0xfd, 0x07, 0x75, 0x9d, 0xc8, 0x9f, 0x8d, 0xc4, 0xd7, + 0x89, 0x34, 0x1b, 0xd3, 0x71, 0x3e, 0xa6, 0xf9, 0x20, 0x85, 0xa5, 0x95, 0x8a, 0x2e, 0xfe, 0x5c, + 0x83, 0xc5, 0xc9, 0xaf, 0x89, 0x4e, 0x41, 0x76, 0x1b, 0xef, 0xf1, 0x2a, 0xe2, 0xf0, 0x68, 0x58, + 0x9b, 0xdd, 0xc6, 0x7b, 0xd2, 0xac, 0x13, 0x2a, 0xfa, 0x16, 0x4c, 0xef, 0x18, 0xfd, 0x10, 0xf3, + 0x25, 0x51, 0xaf, 0xb3, 0x7a, 0xa9, 0x2e, 0xd7, 0x4b, 0x75, 0x77, 0xbb, 0x47, 0x80, 0xba, 0xf0, + 0x48, 0xfd, 0xdd, 0xd0, 0xb0, 0x03, 0x2b, 0xd8, 0x63, 0xcb, 0x85, 0x2a, 0x90, 0x97, 0x0b, 0x05, + 0x5e, 0xcf, 0xbc, 0xaa, 0x2d, 0x7e, 0xa6, 0xc1, 0xb1, 0x89, 0x2f, 0xfd, 0x4d, 0x18, 0xa1, 0xde, + 0x86, 0x29, 0xb2, 0xf0, 0x49, 0x7d, 0xb3, 0x69, 0xf5, 0x36, 0x5f, 0x79, 0x89, 0x0e, 0x27, 0xc7, + 0xca, 0x11, 0x86, 0xc8, 0xe5, 0x08, 0x43, 0x48, 0x8d, 0xd6, 0x77, 0x6e, 0xbe, 0xf2, 0x12, 0x1d, + 0x54, 0x8e, 0x19, 0xa1, 0x80, 0x6c, 0x84, 0x02, 0xfa, 0xaf, 0x72, 0x50, 0x8c, 0x0a, 0x08, 0x69, + 0x0f, 0x6a, 0x77, 0xb5, 0x07, 0x2f, 0x40, 0xc5, 0xc4, 0x26, 0xff, 0xf2, 0x59, 0x8e, 0x2d, 0x76, + 0x73, 0x91, 0x45, 0x57, 0x85, 0xa6, 0xc8, 0x97, 0x13, 0x24, 0x74, 0x06, 0x0a, 0x3c, 0xd1, 0xde, + 0xa3, 0x1b, 0x79, 0xb6, 0xb9, 0x30, 0x1a, 0xd6, 0x90, 0xc0, 0x24, 0xd1, 0x88, 0x0f, 0xb5, 0x00, + 0x58, 0xf5, 0xba, 0x8a, 0x03, 0x83, 0xa7, 0xfc, 0x55, 0xf5, 0x0d, 0xae, 0x46, 0x74, 0x56, 0x87, + 0xc6, 0xfc, 0x72, 0x1d, 0x1a, 0xa3, 0xe8, 0x03, 0x80, 0x81, 0x61, 0xd9, 0x4c, 0x8e, 0xe7, 0xf7, + 0xfa, 0xa4, 0x90, 0xb2, 0x1a, 0x71, 0x32, 0xed, 0xb1, 0xa4, 0xac, 0x3d, 0x46, 0x49, 0xb5, 0xc8, + 0xeb, 0xed, 0x6a, 0x8e, 0xee, 0xd2, 0xa5, 0x49, 0xaa, 0xb9, 0xda, 0xa3, 0xa4, 0x62, 0xe4, 0x22, + 0x92, 0x4e, 0xa1, 0x85, 0x4c, 0x5b, 0xdf, 0xda, 0xc0, 0x81, 0x35, 0xc0, 0x34, 0xb3, 0xe7, 0xd3, + 0x26, 0x30, 0x79, 0xda, 0x04, 0x86, 0x5e, 0x05, 0x30, 0x82, 0x55, 0xc7, 0x0f, 0xae, 0xda, 0x5d, + 0x4c, 0x33, 0xf6, 0x02, 0x1b, 0x7e, 0x8c, 0xca, 0xc3, 0x8f, 0x51, 0xf4, 0x06, 0x94, 0x5c, 0xfe, + 0x11, 0xea, 0xf4, 0x31, 0xcd, 0xc8, 0x0b, 0xec, 0x93, 0x22, 0xc1, 0x92, 0xac, 0xcc, 0x8d, 0xce, + 0x43, 0xb9, 0xeb, 0xd8, 0xdd, 0xd0, 0xf3, 0xb0, 0xdd, 0xdd, 0x5b, 0x33, 0x36, 0x30, 0xcd, 0xbe, + 0x0b, 0x6c, 0xa9, 0x24, 0x48, 0xf2, 0x52, 0x49, 0x90, 0xd0, 0xcb, 0x50, 0x8c, 0xba, 0x17, 0x34, + 0xc1, 0x2e, 0xf2, 0x42, 0x58, 0x80, 0x92, 0x70, 0xcc, 0x49, 0x06, 0x6f, 0xf9, 0x51, 0x96, 0x46, + 0x93, 0x66, 0x3e, 0x78, 0x09, 0x96, 0x07, 0x2f, 0xc1, 0xfa, 0xef, 0x34, 0x98, 0x4f, 0xf3, 0x7b, + 0x62, 0x0d, 0x6a, 0xf7, 0x65, 0x0d, 0xbe, 0x07, 0x05, 0xd7, 0x31, 0xdb, 0xbe, 0x8b, 0xbb, 0x3c, + 0xcc, 0x24, 0x56, 0xe0, 0x35, 0xc7, 0x5c, 0x73, 0x71, 0xf7, 0xff, 0xad, 0x60, 0x73, 0x79, 0xc7, + 0xb1, 0xcc, 0xcb, 0x96, 0xcf, 0x97, 0x8a, 0xcb, 0x28, 0xca, 0x67, 0x3d, 0xcf, 0xc1, 0x66, 0x01, + 0x72, 0xcc, 0x8a, 0xfe, 0xfb, 0x2c, 0x54, 0x92, 0x6b, 0xed, 0xbf, 0xe9, 0x55, 0xd0, 0x0d, 0xc8, + 0x5b, 0x2c, 0xcf, 0xe5, 0x9f, 0xfd, 0xff, 0x91, 0x02, 0x71, 0x3d, 0xee, 0xd2, 0xd5, 0x77, 0x5e, + 0xa8, 0xf3, 0x84, 0x98, 0x4e, 0x01, 0xd5, 0xcc, 0x25, 0x55, 0xcd, 0x1c, 0x44, 0x2d, 0xc8, 0xfb, + 0xd8, 0xdb, 0xb1, 0xba, 0x98, 0x47, 0x94, 0x9a, 0xac, 0xb9, 0xeb, 0x78, 0x98, 0xe8, 0x5c, 0x63, + 0x2c, 0xb1, 0x4e, 0x2e, 0xa3, 0xea, 0xe4, 0x20, 0x7a, 0x0f, 0x8a, 0x5d, 0xc7, 0xde, 0xb0, 0x7a, + 0xab, 0x86, 0xcb, 0x63, 0xca, 0x89, 0x34, 0xad, 0x67, 0x05, 0x13, 0xef, 0x1c, 0x88, 0xc7, 0x44, + 0xe7, 0x20, 0xe2, 0x8a, 0x1d, 0xfa, 0xf7, 0x29, 0x80, 0xd8, 0x39, 0xe8, 0x35, 0x28, 0xe1, 0x5d, + 0xdc, 0x0d, 0x03, 0xc7, 0x13, 0xc1, 0x9d, 0x37, 0xe2, 0x04, 0xac, 0x44, 0x63, 0x88, 0x51, 0xb2, + 0xbb, 0x6c, 0x63, 0x80, 0x7d, 0xd7, 0xe8, 0x8a, 0x0e, 0x1e, 0x1d, 0x4c, 0x04, 0xca, 0xbb, 0x2b, + 0x02, 0xd1, 0xff, 0xc2, 0x14, 0xed, 0xf9, 0xb1, 0xe6, 0x1d, 0x1a, 0x0d, 0x6b, 0x73, 0xb6, 0xda, + 0xed, 0xa3, 0x74, 0xf4, 0x16, 0xcc, 0x6e, 0x47, 0x0b, 0x8f, 0x8c, 0x6d, 0x8a, 0x0a, 0xd0, 0x7c, + 0x2c, 0x26, 0x28, 0xa3, 0x9b, 0x91, 0x71, 0xb4, 0x01, 0x25, 0xc3, 0xb6, 0x9d, 0x80, 0x7e, 0x38, + 0x44, 0x43, 0xef, 0xe9, 0x49, 0xcb, 0xb4, 0xbe, 0x1c, 0xf3, 0xb2, 0xd4, 0x86, 0xee, 0x78, 0x49, + 0x83, 0xbc, 0xe3, 0x25, 0x18, 0xb5, 0x20, 0xd7, 0x37, 0x3a, 0xb8, 0x2f, 0x22, 0xf5, 0x93, 0x13, + 0x4d, 0x5c, 0xa6, 0x6c, 0x4c, 0x3b, 0xfd, 0x4e, 0x33, 0x39, 0xf9, 0x3b, 0xcd, 0x90, 0xc5, 0x0d, + 0xa8, 0x24, 0xc7, 0x73, 0xb0, 0xac, 0xe3, 0x69, 0x39, 0xeb, 0x28, 0xee, 0x9b, 0xe7, 0x18, 0x50, + 0x92, 0x06, 0xf5, 0x20, 0x4c, 0xe8, 0xbf, 0xd0, 0x60, 0x3e, 0x6d, 0xef, 0xa2, 0x55, 0x69, 0xc7, + 0x6b, 0xbc, 0x31, 0x91, 0xb2, 0xd4, 0xb9, 0xec, 0x84, 0xad, 0x1e, 0x6f, 0xf4, 0x26, 0xcc, 0xd9, + 0x8e, 0x89, 0xdb, 0x06, 0x31, 0xd0, 0xb7, 0xfc, 0xa0, 0x9a, 0xa1, 0x0d, 0x5f, 0xda, 0xd0, 0x20, + 0x94, 0x65, 0x41, 0x90, 0xa4, 0x67, 0x15, 0x82, 0xfe, 0x03, 0x0d, 0xca, 0x89, 0x7e, 0xe3, 0x3d, + 0x67, 0x3e, 0x72, 0xbe, 0x92, 0x39, 0x58, 0xbe, 0xa2, 0xff, 0x38, 0x03, 0x25, 0xa9, 0x18, 0xbb, + 0xe7, 0x31, 0x6c, 0x41, 0x99, 0x7f, 0xde, 0x2c, 0xbb, 0xc7, 0x6a, 0xa0, 0x0c, 0xef, 0x2c, 0x8c, + 0xb5, 0xf7, 0x2f, 0x39, 0x9d, 0xb5, 0x88, 0x97, 0x96, 0x40, 0xb4, 0xed, 0xe4, 0x2b, 0x98, 0x64, + 0x62, 0x4e, 0xa5, 0xa0, 0x1b, 0xb0, 0x10, 0xba, 0xa6, 0x11, 0xe0, 0xb6, 0xcf, 0x1b, 0xe5, 0x6d, + 0x3b, 0x1c, 0x74, 0xb0, 0x47, 0x77, 0xfc, 0x34, 0x6b, 0x94, 0x30, 0x0e, 0xd1, 0x49, 0xbf, 0x42, + 0xe9, 0x92, 0xce, 0xf9, 0x34, 0xba, 0x7e, 0x01, 0xd0, 0x78, 0x33, 0x58, 0x99, 0x5f, 0xed, 0x80, + 0xf3, 0xfb, 0x89, 0x06, 0x95, 0x64, 0x8f, 0xf7, 0xa1, 0x38, 0x7a, 0x0f, 0x8a, 0x51, 0xbf, 0xf6, + 0x9e, 0x07, 0xf0, 0x2c, 0xe4, 0x3c, 0x6c, 0xf8, 0x8e, 0xcd, 0x77, 0x26, 0x0d, 0x31, 0x0c, 0x91, + 0x43, 0x0c, 0x43, 0xf4, 0xeb, 0x30, 0xc3, 0x66, 0xf0, 0x6d, 0xab, 0x1f, 0x60, 0x0f, 0x9d, 0x83, + 0x9c, 0x1f, 0x18, 0x01, 0xf6, 0xab, 0xda, 0xc9, 0xec, 0xe9, 0xb9, 0x33, 0x0b, 0xe3, 0xad, 0x59, + 0x42, 0x66, 0x5a, 0x19, 0xa7, 0xac, 0x95, 0x21, 0xfa, 0xf7, 0x34, 0x98, 0x91, 0x3b, 0xd0, 0xf7, + 0x47, 0xed, 0x1d, 0xbe, 0xda, 0xc7, 0x62, 0x0c, 0xfd, 0xfb, 0xe3, 0xd9, 0x3b, 0xb3, 0xfe, 0x6b, + 0x8d, 0xcd, 0x6c, 0xd4, 0xba, 0xbc, 0x57, 0xf3, 0xbd, 0xb8, 0x7f, 0x41, 0x76, 0x98, 0x4f, 0x03, + 0xdb, 0x41, 0xfb, 0x17, 0x34, 0xfc, 0x29, 0xe2, 0x72, 0xf8, 0x53, 0x08, 0xfa, 0x1f, 0x33, 0x74, + 0xe4, 0x71, 0x9b, 0xfa, 0x61, 0x77, 0x6e, 0x12, 0xd9, 0x49, 0xf6, 0x0e, 0xb2, 0x93, 0xe7, 0x20, + 0x4f, 0x3f, 0x07, 0x51, 0xe2, 0x40, 0x9d, 0x46, 0x20, 0xf5, 0x98, 0x90, 0x21, 0xb7, 0x89, 0x5a, + 0xd3, 0xf7, 0x18, 0xb5, 0xfe, 0xa9, 0xc1, 0x9c, 0xda, 0xc7, 0x7f, 0xe8, 0xd3, 0x3a, 0xb6, 0xa0, + 0xb2, 0x0f, 0x68, 0x41, 0xfd, 0x43, 0x83, 0x59, 0xe5, 0x78, 0xe1, 0xd1, 0x79, 0xf5, 0x9f, 0x66, + 0x60, 0x21, 0x5d, 0xcd, 0x03, 0x29, 0x9f, 0x2e, 0x00, 0x49, 0x84, 0x2e, 0xc6, 0x5f, 0xf6, 0xa3, + 0x63, 0xd5, 0x13, 0x7d, 0x05, 0x91, 0x45, 0x8d, 0x9d, 0x0b, 0x08, 0x71, 0x74, 0x03, 0x4a, 0x96, + 0x74, 0x02, 0x91, 0x4d, 0x6b, 0x14, 0xcb, 0xe7, 0x0e, 0xac, 0x30, 0x9e, 0x70, 0xda, 0x20, 0xab, + 0x6a, 0xe6, 0x60, 0x8a, 0xa4, 0x1e, 0xfa, 0x0e, 0xe4, 0xf9, 0x70, 0xd0, 0x8b, 0x50, 0xa4, 0xbb, + 0x94, 0x56, 0x04, 0x2c, 0xed, 0xa4, 0x1f, 0x4d, 0x02, 0x26, 0xee, 0x00, 0x14, 0x04, 0x86, 0x5e, + 0x01, 0x20, 0x89, 0x23, 0xdf, 0x9f, 0x19, 0xba, 0x3f, 0x69, 0xe5, 0xe1, 0x3a, 0xe6, 0xd8, 0xa6, + 0x2c, 0x46, 0xa0, 0xfe, 0xcb, 0x0c, 0x94, 0xe4, 0x33, 0x8f, 0xbb, 0x32, 0xfe, 0x31, 0x88, 0xaa, + 0xb0, 0x6d, 0x98, 0x26, 0xf9, 0x8b, 0x45, 0x40, 0x6e, 0x4c, 0x9c, 0x24, 0xf1, 0xff, 0xb2, 0x90, + 0x60, 0x35, 0x00, 0x3d, 0x55, 0xb6, 0x12, 0x24, 0xc9, 0x6a, 0x25, 0x49, 0x5b, 0xdc, 0x86, 0xa3, + 0xa9, 0xaa, 0xe4, 0xcc, 0x7d, 0xfa, 0x7e, 0x65, 0xee, 0xbf, 0x99, 0x86, 0xa3, 0xa9, 0x67, 0x4d, + 0x0f, 0x7d, 0x17, 0xab, 0x3b, 0x28, 0x7b, 0x5f, 0x76, 0xd0, 0x27, 0x5a, 0x9a, 0x67, 0x59, 0xdf, + 0xfe, 0xb5, 0x03, 0x1c, 0xc0, 0xdd, 0x2f, 0x1f, 0xab, 0xcb, 0x72, 0xfa, 0xae, 0xf6, 0x44, 0xee, + 0xa0, 0x7b, 0x02, 0x3d, 0xcf, 0x8a, 0x30, 0x6a, 0x2b, 0x4f, 0x6d, 0x89, 0x08, 0x91, 0x30, 0x95, + 0xe7, 0x10, 0xa9, 0xcb, 0x85, 0x04, 0x2b, 0xfd, 0x0b, 0x71, 0x5d, 0xce, 0x79, 0x92, 0xd5, 0xff, + 0x8c, 0x8c, 0xff, 0x67, 0xd7, 0xf0, 0xbf, 0x34, 0x28, 0x27, 0x0e, 0x9f, 0x1f, 0x9d, 0x6f, 0xd0, + 0xa7, 0x1a, 0x14, 0xa3, 0x7b, 0x0f, 0xf7, 0x9c, 0x86, 0x2e, 0x43, 0x0e, 0xb3, 0xb3, 0x77, 0x16, + 0xee, 0x8e, 0x24, 0xee, 0x46, 0x11, 0x1a, 0xbf, 0x0d, 0x95, 0x38, 0x6e, 0x6f, 0x71, 0x41, 0xfd, + 0x0f, 0x9a, 0x48, 0x30, 0xe3, 0x31, 0x3d, 0x54, 0x57, 0xc4, 0xef, 0x94, 0xbd, 0xdb, 0x77, 0xfa, + 0x6d, 0x11, 0xa6, 0x29, 0x1f, 0x29, 0x00, 0x03, 0xec, 0x0d, 0x2c, 0xdb, 0xe8, 0xd3, 0xd7, 0x29, + 0xb0, 0x7d, 0x2b, 0x30, 0x79, 0xdf, 0x0a, 0x0c, 0x6d, 0x42, 0x39, 0x6e, 0x5a, 0x51, 0x35, 0xe9, + 0x57, 0xae, 0xde, 0x51, 0x99, 0x58, 0x2b, 0x3c, 0x21, 0xa9, 0x9e, 0x49, 0x27, 0x88, 0xc8, 0x84, + 0xb9, 0xae, 0x63, 0x07, 0x86, 0x65, 0x63, 0x8f, 0x19, 0xca, 0xa6, 0x5d, 0x39, 0x39, 0xab, 0xf0, + 0xb0, 0xda, 0x5f, 0x95, 0x53, 0xaf, 0x9c, 0xa8, 0x34, 0xf4, 0x21, 0xcc, 0x8a, 0x24, 0x9c, 0x19, + 0x99, 0x4a, 0xbb, 0x72, 0xb2, 0x22, 0xb3, 0xb0, 0x25, 0xad, 0x48, 0xa9, 0x57, 0x4e, 0x14, 0x12, + 0xea, 0x43, 0xc5, 0x75, 0xcc, 0x75, 0x9b, 0xb7, 0x1d, 0x8c, 0x4e, 0x1f, 0xf3, 0x4e, 0xe9, 0xd2, + 0x58, 0xca, 0xa3, 0x70, 0xb1, 0x50, 0x9c, 0x94, 0x55, 0x2f, 0x71, 0x25, 0xa9, 0xe8, 0x03, 0x98, + 0xe9, 0x93, 0x5a, 0x68, 0x65, 0xd7, 0xb5, 0x3c, 0x6c, 0xa6, 0x5f, 0xb9, 0xba, 0x2c, 0x71, 0xb0, + 0x40, 0x28, 0xcb, 0xa8, 0xd7, 0x4e, 0x64, 0x0a, 0xf1, 0xfe, 0xc0, 0xd8, 0x6d, 0x85, 0xb6, 0xbf, + 0xb2, 0xcb, 0xaf, 0xcf, 0xe4, 0xd3, 0xbc, 0xbf, 0xaa, 0x32, 0x31, 0xef, 0x27, 0x24, 0x55, 0xef, + 0x27, 0x88, 0xe8, 0x32, 0x8d, 0xf3, 0xcc, 0x25, 0xec, 0xea, 0xd5, 0xc2, 0xd8, 0x6c, 0x31, 0x6f, + 0xb0, 0xa6, 0x05, 0x7f, 0x52, 0x94, 0x46, 0x1a, 0xb8, 0x0f, 0xe8, 0x6b, 0xb7, 0x70, 0x10, 0x7a, + 0x36, 0x36, 0xf9, 0xad, 0xab, 0x71, 0x1f, 0x28, 0x5c, 0x91, 0x0f, 0x14, 0x74, 0xcc, 0x07, 0x0a, + 0x95, 0xac, 0x29, 0xd7, 0x31, 0xaf, 0xb3, 0x2d, 0x13, 0x44, 0x77, 0xb1, 0x1e, 0x1f, 0x33, 0x15, + 0xb3, 0xb0, 0x35, 0xa5, 0x48, 0xa9, 0x6b, 0x4a, 0x21, 0xf1, 0xeb, 0x3f, 0xf2, 0x65, 0x11, 0x36, + 0x53, 0xa5, 0x09, 0xd7, 0x7f, 0xc6, 0x38, 0xa3, 0xeb, 0x3f, 0x63, 0x94, 0xb1, 0xeb, 0x3f, 0x63, + 0x1c, 0xc4, 0x7a, 0xcf, 0xb0, 0x7b, 0x97, 0x9c, 0x8e, 0xba, 0xaa, 0x67, 0xd2, 0xac, 0x9f, 0x4f, + 0xe1, 0x64, 0xd6, 0xd3, 0x74, 0xa8, 0xd6, 0xd3, 0x38, 0x9a, 0x05, 0xd1, 0xdc, 0xd0, 0x3f, 0xd3, + 0xa0, 0x9c, 0x88, 0x33, 0xe8, 0x4d, 0x88, 0x2e, 0x39, 0x5c, 0xdf, 0x73, 0x45, 0x9a, 0xac, 0x5c, + 0x8a, 0x20, 0x78, 0xda, 0xa5, 0x08, 0x82, 0xa3, 0xcb, 0x00, 0xd1, 0x37, 0xe9, 0x76, 0x41, 0x9a, + 0xe6, 0x68, 0x31, 0xa7, 0x9c, 0xa3, 0xc5, 0xa8, 0xfe, 0x65, 0x16, 0x0a, 0x62, 0xa1, 0x3e, 0x90, + 0x32, 0xaa, 0x01, 0xf9, 0x01, 0xf6, 0xe9, 0xe5, 0x88, 0x4c, 0x9c, 0x0d, 0x71, 0x48, 0xce, 0x86, + 0x38, 0xa4, 0x26, 0x6b, 0xd9, 0xbb, 0x4a, 0xd6, 0xa6, 0x0e, 0x9c, 0xac, 0x61, 0x7a, 0x30, 0x2a, + 0x85, 0x5b, 0x71, 0xaa, 0x71, 0xfb, 0x18, 0x2e, 0x8e, 0x4d, 0x65, 0xc1, 0xc4, 0xb1, 0xa9, 0x4c, + 0x42, 0xdb, 0x70, 0x58, 0x3a, 0x79, 0xe1, 0x9d, 0x2f, 0x12, 0xf8, 0xe6, 0x26, 0x9f, 0x42, 0xb7, + 0x28, 0x17, 0xdb, 0xde, 0xdb, 0x09, 0x54, 0xce, 0x76, 0x93, 0x34, 0xfd, 0xaf, 0x19, 0x98, 0x53, + 0xc7, 0xfb, 0x40, 0x1c, 0xfb, 0x22, 0x14, 0xf1, 0xae, 0x15, 0xb4, 0xbb, 0x8e, 0x89, 0x79, 0xc9, + 0x48, 0xfd, 0x44, 0xc0, 0xb3, 0x8e, 0xa9, 0xf8, 0x49, 0x60, 0xf2, 0x6a, 0xc8, 0x1e, 0x68, 0x35, + 0xc4, 0x8d, 0xc2, 0xa9, 0xfd, 0x1b, 0x85, 0xe9, 0xf3, 0x5c, 0x7c, 0x40, 0xf3, 0x7c, 0x2b, 0x03, + 0x95, 0x64, 0x34, 0xfe, 0x66, 0x6c, 0x21, 0x75, 0x37, 0x64, 0x0f, 0xbc, 0x1b, 0xde, 0x82, 0x59, + 0x92, 0x3b, 0x1a, 0x41, 0xc0, 0xaf, 0x0d, 0x4e, 0xd1, 0x9c, 0x8b, 0xc5, 0xa6, 0xd0, 0x5e, 0x16, + 0xb8, 0x12, 0x9b, 0x24, 0x5c, 0xff, 0x6e, 0x06, 0x66, 0x95, 0xaf, 0xc6, 0xa3, 0x17, 0x52, 0xf4, + 0x32, 0xcc, 0x2a, 0xc9, 0x98, 0xfe, 0x7d, 0xb6, 0x4e, 0xd4, 0x2c, 0xe8, 0xd1, 0x9b, 0x97, 0x39, + 0x98, 0x91, 0xb3, 0x3a, 0xbd, 0x09, 0xe5, 0x44, 0x12, 0x26, 0xbf, 0x80, 0x76, 0x90, 0x17, 0xd0, + 0x17, 0x60, 0x3e, 0x2d, 0x77, 0xd0, 0xcf, 0xc3, 0x7c, 0xda, 0x57, 0xfd, 0xce, 0x0d, 0x7c, 0xae, + 0x51, 0x0b, 0xe3, 0x17, 0x8c, 0x2f, 0x00, 0xd8, 0xf8, 0x66, 0x7b, 0xdf, 0xf2, 0x8f, 0xcd, 0x27, + 0xbe, 0x79, 0x29, 0x51, 0x2d, 0x15, 0x04, 0x46, 0x34, 0x39, 0x7d, 0xb3, 0xbd, 0x6f, 0xd1, 0x45, + 0x35, 0x39, 0x7d, 0x73, 0x4c, 0x93, 0xc0, 0xf4, 0x1f, 0x66, 0x45, 0x65, 0x1e, 0xdf, 0xd0, 0x7d, + 0x1f, 0x2a, 0xae, 0x78, 0xd8, 0x7f, 0xb4, 0xb4, 0x36, 0x89, 0xf8, 0x93, 0x96, 0xe6, 0x54, 0x8a, + 0xaa, 0x9b, 0x17, 0x9d, 0x99, 0x03, 0xea, 0x6e, 0x25, 0xaa, 0xcf, 0x39, 0x95, 0x82, 0xbe, 0x0d, + 0x87, 0xc5, 0x05, 0xa6, 0x1d, 0x2c, 0x06, 0x9e, 0x9d, 0xa8, 0x9c, 0x5d, 0x28, 0x8e, 0x04, 0x92, + 0x23, 0x2f, 0x27, 0x48, 0x09, 0xf5, 0x7c, 0xec, 0x53, 0x07, 0x55, 0x9f, 0x1c, 0x7c, 0x39, 0x41, + 0xd2, 0x3f, 0xd5, 0xa0, 0x9c, 0xb8, 0xf3, 0x8c, 0xce, 0x41, 0x81, 0xfe, 0x24, 0xea, 0xf6, 0x1e, + 0xa0, 0x0b, 0x92, 0xf2, 0x29, 0x16, 0xf2, 0x1c, 0x42, 0x2f, 0x43, 0x31, 0xba, 0x1a, 0xcd, 0xcf, + 0x44, 0xd9, 0xe6, 0x13, 0xa0, 0xb2, 0xf9, 0x04, 0xa8, 0xff, 0x4c, 0x83, 0x63, 0x13, 0xef, 0x43, + 0x3f, 0xec, 0x9e, 0xc1, 0x33, 0xcf, 0x43, 0x41, 0x9c, 0x5a, 0x22, 0x80, 0xdc, 0xbb, 0xeb, 0x2b, + 0xeb, 0x2b, 0xe7, 0x2a, 0x87, 0x50, 0x09, 0xf2, 0xd7, 0x56, 0xae, 0x9c, 0xbb, 0x78, 0xe5, 0x7c, + 0x45, 0x23, 0x0f, 0xad, 0xf5, 0x2b, 0x57, 0xc8, 0x43, 0xe6, 0x99, 0xcb, 0xf2, 0x1d, 0x2a, 0xf6, + 0x3d, 0x46, 0x33, 0x50, 0x58, 0x76, 0x5d, 0x1a, 0x00, 0x98, 0xec, 0xca, 0x8e, 0x45, 0xf6, 0x6a, + 0x45, 0x43, 0x79, 0xc8, 0x5e, 0xbd, 0xba, 0x5a, 0xc9, 0xa0, 0x79, 0xa8, 0x9c, 0xc3, 0x86, 0xd9, + 0xb7, 0x6c, 0x2c, 0xa2, 0x4e, 0x25, 0xdb, 0xdc, 0xfa, 0xe2, 0xab, 0x25, 0xed, 0xcb, 0xaf, 0x96, + 0xb4, 0xbf, 0x7c, 0xb5, 0xa4, 0xdd, 0xfa, 0x7a, 0xe9, 0xd0, 0x97, 0x5f, 0x2f, 0x1d, 0xfa, 0xd3, + 0xd7, 0x4b, 0x87, 0xde, 0x7f, 0x5e, 0xfa, 0xf9, 0x1f, 0x7b, 0x27, 0xd7, 0x73, 0x48, 0xc0, 0xe5, + 0x4f, 0x8d, 0xe4, 0x0f, 0x1e, 0x3f, 0xcf, 0x9c, 0x58, 0xa6, 0x8f, 0xd7, 0x18, 0x5f, 0xfd, 0xa2, + 0x53, 0x67, 0x00, 0xfd, 0xcd, 0x9a, 0xdf, 0xc9, 0xd1, 0xdf, 0xa6, 0xbd, 0xf8, 0xef, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x21, 0x3d, 0x8d, 0x38, 0x2b, 0x39, 0x00, 0x00, } func (m *EventSequence) Marshal() (dAtA []byte, err error) { @@ -6073,6 +6133,27 @@ func (m *Error_JobRunPreemptedError) MarshalToSizedBuffer(dAtA []byte) (int, err } return len(dAtA) - i, nil } +func (m *Error_GangJobUnschedulable) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Error_GangJobUnschedulable) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.GangJobUnschedulable != nil { + { + size, err := m.GangJobUnschedulable.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x62 + } + return len(dAtA) - i, nil +} func (m *KubernetesError) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6511,6 +6592,36 @@ func (m *JobRunPreemptedError) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *GangJobUnschedulable) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GangJobUnschedulable) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GangJobUnschedulable) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Message) > 0 { + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *JobDuplicateDetected) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -7835,6 +7946,18 @@ func (m *Error_JobRunPreemptedError) Size() (n int) { } return n } +func (m *Error_GangJobUnschedulable) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.GangJobUnschedulable != nil { + l = m.GangJobUnschedulable.Size() + n += 1 + l + sovEvents(uint64(l)) + } + return n +} func (m *KubernetesError) Size() (n int) { if m == nil { return 0 @@ -8023,6 +8146,19 @@ func (m *JobRunPreemptedError) Size() (n int) { return n } +func (m *GangJobUnschedulable) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Message) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + return n +} + func (m *JobDuplicateDetected) Size() (n int) { if m == nil { return 0 @@ -14126,6 +14262,41 @@ func (m *Error) Unmarshal(dAtA []byte) error { } m.Reason = &Error_JobRunPreemptedError{v} iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GangJobUnschedulable", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &GangJobUnschedulable{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Reason = &Error_GangJobUnschedulable{v} + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipEvents(dAtA[iNdEx:]) @@ -15402,6 +15573,88 @@ func (m *JobRunPreemptedError) Unmarshal(dAtA []byte) error { } return nil } +func (m *GangJobUnschedulable) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GangJobUnschedulable: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GangJobUnschedulable: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *JobDuplicateDetected) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/armadaevents/events.proto b/pkg/armadaevents/events.proto index 448bdecab3f..f2b890dc1ac 100644 --- a/pkg/armadaevents/events.proto +++ b/pkg/armadaevents/events.proto @@ -404,6 +404,7 @@ message Error { PodLeaseReturned podLeaseReturned = 9; PodTerminated podTerminated = 10; JobRunPreemptedError jobRunPreemptedError = 11; + GangJobUnschedulable gangJobUnschedulable = 12; } } @@ -486,6 +487,10 @@ message MaxRunsExceeded { message JobRunPreemptedError{ } +message GangJobUnschedulable{ + string message = 1; +} + // Generated by the scheduler whenever it detects a SubmitJob message that includes a previously used deduplication id // (i.e., when it detects a duplicate job submission). message JobDuplicateDetected { diff --git a/third_party/airflow/armada/operators/armada_deferrable.py b/third_party/airflow/armada/operators/armada_deferrable.py index 2f53a702228..f7aa1413637 100644 --- a/third_party/airflow/armada/operators/armada_deferrable.py +++ b/third_party/airflow/armada/operators/armada_deferrable.py @@ -103,6 +103,25 @@ def __init__( self.lookout_url_template = lookout_url_template self.poll_interval = poll_interval + def serialize(self) -> dict: + """ + Get a serialized version of this object. + + :return: A dict of keyword arguments used when instantiating + this object. + """ + + return { + "task_id": self.task_id, + "name": self.name, + "armada_channel_args": self.armada_channel_args.serialize(), + "job_service_channel_args": self.job_service_channel_args.serialize(), + "armada_queue": self.armada_queue, + "job_request_items": self.job_request_items, + "lookout_url_template": self.lookout_url_template, + "poll_interval": self.poll_interval, + } + def execute(self, context) -> None: """ Executes the Armada Operator. Only meant to be called by airflow. @@ -156,6 +175,7 @@ def execute(self, context) -> None: armada_queue=self.armada_queue, job_set_id=context["run_id"], airflow_task_name=self.name, + poll_interval=self.poll_interval, ), method_name="resume_job_complete", kwargs={"job_id": job_id}, @@ -216,6 +236,7 @@ class ArmadaJobCompleteTrigger(BaseTrigger): :param job_set_id: The ID of the job set. :param airflow_task_name: Name of the airflow task to which this trigger belongs. + :param poll_interval: How often to poll jobservice to get status. :return: An armada job complete trigger instance. """ @@ -226,6 +247,7 @@ def __init__( armada_queue: str, job_set_id: str, airflow_task_name: str, + poll_interval: int = 30, ) -> None: super().__init__() self.job_id = job_id @@ -233,6 +255,7 @@ def __init__( self.armada_queue = armada_queue self.job_set_id = job_set_id self.airflow_task_name = airflow_task_name + self.poll_interval = poll_interval def serialize(self) -> tuple: return ( @@ -243,9 +266,21 @@ def serialize(self) -> tuple: "armada_queue": self.armada_queue, "job_set_id": self.job_set_id, "airflow_task_name": self.airflow_task_name, + "poll_interval": self.poll_interval, }, ) + def __eq__(self, o): + return ( + self.task_id == o.task_id + and self.job_id == o.job_id + and self.job_service_channel_args == o.job_service_channel_args + and self.armada_queue == o.armada_queue + and self.job_set_id == o.job_set_id + and self.airflow_task_name == o.airflow_task_name + and self.poll_interval == o.poll_interval + ) + async def run(self): """ Runs the trigger. Meant to be called by an airflow triggerer process. @@ -255,12 +290,12 @@ async def run(self): ) job_state, job_message = await search_for_job_complete_async( - job_service_client=job_service_client, armada_queue=self.armada_queue, job_set_id=self.job_set_id, airflow_task_name=self.airflow_task_name, job_id=self.job_id, - poll_interval=self.poll_interval, + job_service_client=job_service_client, log=self.log, + poll_interval=self.poll_interval, ) yield TriggerEvent({"job_state": job_state, "job_message": job_message}) diff --git a/third_party/airflow/armada/operators/utils.py b/third_party/airflow/armada/operators/utils.py index e3c68beb321..1ab7fa35d04 100644 --- a/third_party/airflow/armada/operators/utils.py +++ b/third_party/airflow/armada/operators/utils.py @@ -217,6 +217,7 @@ async def search_for_job_complete_async( job_id: str, job_service_client: JobServiceAsyncIOClient, log, + poll_interval: int, time_out_for_failure: int = 7200, ) -> Tuple[JobState, str]: """ @@ -231,6 +232,7 @@ async def search_for_job_complete_async( :param job_id: The name of the job id that armada assigns to it :param job_service_client: A JobServiceClient that is used for polling. It is optional only for testing + :param poll_interval: How often to poll jobservice to get status. :param time_out_for_failure: The amount of time a job can be in job_id_not_found before we decide it was a invalid job @@ -251,7 +253,7 @@ async def search_for_job_complete_async( job_state = job_state_from_pb(job_status_return.state) log.debug(f"Got job state '{job_state.name}' for job {job_id}") - await asyncio.sleep(3) + await asyncio.sleep(poll_interval) if job_state == JobState.SUCCEEDED: job_message = f"Armada {airflow_task_name}:{job_id} succeeded" diff --git a/third_party/airflow/examples/big_armada.py b/third_party/airflow/examples/big_armada.py index f1196307227..dc64cdc76b2 100644 --- a/third_party/airflow/examples/big_armada.py +++ b/third_party/airflow/examples/big_armada.py @@ -57,7 +57,7 @@ def submit_sleep_job(): with DAG( dag_id="big_armada", start_date=pendulum.datetime(2016, 1, 1, tz="UTC"), - schedule_interval="@daily", + schedule="@daily", catchup=False, default_args={"retries": 2}, ) as dag: diff --git a/third_party/airflow/pyproject.toml b/third_party/airflow/pyproject.toml index bd9814cc10c..d3fb7abfa6f 100644 --- a/third_party/airflow/pyproject.toml +++ b/third_party/airflow/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "armada_airflow" -version = "0.5.3" +version = "0.5.6" description = "Armada Airflow Operator" requires-python = ">=3.7" # Note(JayF): This dependency value is not suitable for release. Whatever @@ -10,9 +10,9 @@ requires-python = ">=3.7" dependencies = [ "armada-client", "apache-airflow>=2.6.3", - "grpcio>=1.46.3", - "grpcio-tools>=1.46.3", - "types-protobuf>=3.19.22" + "grpcio==1.58.0", + "grpcio-tools==1.58.0", + "types-protobuf==4.24.0.1" ] authors = [{name = "Armada-GROSS", email = "armada@armadaproject.io"}] license = { text = "Apache Software License" } @@ -20,7 +20,7 @@ readme = "README.md" [project.optional-dependencies] format = ["black==23.7.0", "flake8==6.1.0", "pylint==2.17.5"] -test = ["pytest==7.3.1", "coverage>=6.5.0", "pytest-asyncio==0.21.1"] +test = ["pytest==7.3.1", "coverage==7.3.1", "pytest-asyncio==0.21.1"] # note(JayF): sphinx-jekyll-builder was broken by sphinx-markdown-builder 0.6 -- so pin to 0.5.5 docs = ["sphinx==7.1.2", "sphinx-jekyll-builder==0.3.0", "sphinx-toolbox==3.2.0b1", "sphinx-markdown-builder==0.5.5"] diff --git a/third_party/airflow/tests/unit/test_airflow_operator_mock.py b/third_party/airflow/tests/unit/test_airflow_operator_mock.py index 4634e644795..1ab2d37ced1 100644 --- a/third_party/airflow/tests/unit/test_airflow_operator_mock.py +++ b/third_party/airflow/tests/unit/test_airflow_operator_mock.py @@ -170,7 +170,7 @@ def test_annotate_job_request_items(): dag = DAG( dag_id="hello_armada", start_date=pendulum.datetime(2016, 1, 1, tz="UTC"), - schedule_interval="@daily", + schedule="@daily", catchup=False, default_args={"retries": 2}, ) @@ -204,7 +204,7 @@ def test_parameterize_armada_operator(): dag = DAG( dag_id="hello_armada", start_date=pendulum.datetime(2016, 1, 1, tz="UTC"), - schedule_interval="@daily", + schedule="@daily", catchup=False, default_args={"retries": 2}, ) diff --git a/third_party/airflow/tests/unit/test_armada_deferrable_operator.py b/third_party/airflow/tests/unit/test_armada_deferrable_operator.py new file mode 100644 index 00000000000..0f156ed177e --- /dev/null +++ b/third_party/airflow/tests/unit/test_armada_deferrable_operator.py @@ -0,0 +1,171 @@ +import copy + +import pytest + +from armada_client.armada import submit_pb2 +from armada_client.k8s.io.api.core.v1 import generated_pb2 as core_v1 +from armada_client.k8s.io.apimachinery.pkg.api.resource import ( + generated_pb2 as api_resource, +) +from armada.operators.armada_deferrable import ArmadaDeferrableOperator +from armada.operators.grpc import CredentialsCallback + + +def test_serialize_armada_deferrable(): + grpc_chan_args = { + "target": "localhost:443", + "credentials_callback_args": { + "module_name": "channel_test", + "function_name": "get_credentials", + "function_kwargs": { + "example_arg": "test", + }, + }, + } + + pod = core_v1.PodSpec( + containers=[ + core_v1.Container( + name="sleep", + image="busybox", + args=["sleep", "10s"], + securityContext=core_v1.SecurityContext(runAsUser=1000), + resources=core_v1.ResourceRequirements( + requests={ + "cpu": api_resource.Quantity(string="120m"), + "memory": api_resource.Quantity(string="510Mi"), + }, + limits={ + "cpu": api_resource.Quantity(string="120m"), + "memory": api_resource.Quantity(string="510Mi"), + }, + ), + ) + ], + ) + + job_requests = [ + submit_pb2.JobSubmitRequestItem( + priority=1, + pod_spec=pod, + namespace="personal-anonymous", + annotations={"armadaproject.io/hello": "world"}, + ) + ] + + source = ArmadaDeferrableOperator( + task_id="test_task_id", + name="test task", + armada_channel_args=grpc_chan_args, + job_service_channel_args=grpc_chan_args, + armada_queue="test-queue", + job_request_items=job_requests, + lookout_url_template="https://lookout.test.domain/", + poll_interval=5, + ) + + serialized = source.serialize() + assert serialized["name"] == source.name + + reconstituted = ArmadaDeferrableOperator(**serialized) + assert reconstituted == source + + +get_lookout_url_test_cases = [ + ( + "http://localhost:8089/jobs?job_id=", + "test_id", + "http://localhost:8089/jobs?job_id=test_id", + ), + ( + "https://lookout.armada.domain/jobs?job_id=", + "test_id", + "https://lookout.armada.domain/jobs?job_id=test_id", + ), + ("", "test_id", ""), + (None, "test_id", ""), +] + + +@pytest.mark.parametrize( + "lookout_url_template, job_id, expected_url", get_lookout_url_test_cases +) +def test_get_lookout_url(lookout_url_template, job_id, expected_url): + armada_channel_args = {"target": "127.0.0.1:50051"} + job_service_channel_args = {"target": "127.0.0.1:60003"} + + operator = ArmadaDeferrableOperator( + task_id="test_task_id", + name="test_task", + armada_channel_args=armada_channel_args, + job_service_channel_args=job_service_channel_args, + armada_queue="test_queue", + job_request_items=[], + lookout_url_template=lookout_url_template, + ) + + assert operator._get_lookout_url(job_id) == expected_url + + +def test_deepcopy_operator(): + armada_channel_args = {"target": "127.0.0.1:50051"} + job_service_channel_args = {"target": "127.0.0.1:60003"} + + operator = ArmadaDeferrableOperator( + task_id="test_task_id", + name="test_task", + armada_channel_args=armada_channel_args, + job_service_channel_args=job_service_channel_args, + armada_queue="test_queue", + job_request_items=[], + lookout_url_template="http://localhost:8089/jobs?job_id=", + ) + + try: + copy.deepcopy(operator) + except Exception as e: + assert False, f"{e}" + + +def test_deepcopy_operator_with_grpc_credentials_callback(): + armada_channel_args = { + "target": "127.0.0.1:50051", + "credentials_callback_args": { + "module_name": "tests.unit.test_armada_operator", + "function_name": "__example_test_callback", + "function_kwargs": { + "test_arg": "fake_arg", + }, + }, + } + job_service_channel_args = {"target": "127.0.0.1:60003"} + + operator = ArmadaDeferrableOperator( + task_id="test_task_id", + name="test_task", + armada_channel_args=armada_channel_args, + job_service_channel_args=job_service_channel_args, + armada_queue="test_queue", + job_request_items=[], + lookout_url_template="http://localhost:8089/jobs?job_id=", + ) + + try: + copy.deepcopy(operator) + except Exception as e: + assert False, f"{e}" + + +def __example_test_callback(foo=None): + return f"fake_cred {foo}" + + +def test_credentials_callback(): + callback = CredentialsCallback( + module_name="test_armada_operator", + function_name="__example_test_callback", + function_kwargs={"foo": "bar"}, + ) + + result = callback.call() + assert result == "fake_cred bar" diff --git a/third_party/airflow/tests/unit/test_search_for_job_complete_asyncio.py b/third_party/airflow/tests/unit/test_search_for_job_complete_asyncio.py index 83cc3e220aa..a842fa994d3 100644 --- a/third_party/airflow/tests/unit/test_search_for_job_complete_asyncio.py +++ b/third_party/airflow/tests/unit/test_search_for_job_complete_asyncio.py @@ -71,6 +71,7 @@ async def test_failed_event(js_aio_client): job_service_client=js_aio_client, time_out_for_failure=5, log=logging.getLogger(), + poll_interval=1, ) assert job_complete[0] == JobState.FAILED assert ( @@ -89,6 +90,7 @@ async def test_successful_event(js_aio_client): job_service_client=js_aio_client, time_out_for_failure=5, log=logging.getLogger(), + poll_interval=1, ) assert job_complete[0] == JobState.SUCCEEDED assert job_complete[1] == "Armada test:test_succeeded succeeded" @@ -104,6 +106,7 @@ async def test_cancelled_event(js_aio_client): job_service_client=js_aio_client, time_out_for_failure=5, log=logging.getLogger(), + poll_interval=1, ) assert job_complete[0] == JobState.CANCELLED assert job_complete[1] == "Armada test:test_cancelled cancelled" @@ -119,6 +122,7 @@ async def test_job_id_not_found(js_aio_client): time_out_for_failure=5, job_service_client=js_aio_client, log=logging.getLogger(), + poll_interval=1, ) assert job_complete[0] == JobState.JOB_ID_NOT_FOUND assert ( @@ -142,6 +146,7 @@ async def test_error_retry(js_aio_retry_client): job_service_client=js_aio_retry_client, time_out_for_failure=5, log=logging.getLogger(), + poll_interval=1, ) assert job_complete[0] == JobState.SUCCEEDED assert job_complete[1] == "Armada test:test_succeeded succeeded"