scheduler: allow controlling num_schedulers via API

api/operator.go

@@ -207,6 +207,18 @@ type SchedulerSetConfigurationResponse struct {
 	WriteMeta
 }
 
+// SetNumSchedulersRequest is used to set the number of schedulers
+type SetNumSchedulersRequest struct {
+	Schedulers int
+	WriteRequest
+}
+
+// GetNumSchedulersResponse is used to get number of schedulers
+type GetNumSchedulersResponse struct {
+	Schedulers int
+	QueryMeta
+}
+
 // SchedulerAlgorithm is an enum string that encapsulates the valid options for a
 // SchedulerConfiguration block's SchedulerAlgorithm. These modes will allow the
 // scheduler to be user-selectable.
@@ -235,6 +247,16 @@ func (op *Operator) SchedulerGetConfiguration(q *QueryOptions) (*SchedulerConfig
 	return &resp, qm, nil
 }
 
+// SchedulerGetNumSchedulers is used to query the current number of schedulers
+func (op *Operator) SchedulerGetNumSchedulers(q *QueryOptions) (*GetNumSchedulersResponse, *QueryMeta, error) {
+	var resp GetNumSchedulersResponse
+	qm, err := op.c.query("/v1/operator/scheduler/schedulers", &resp, q)
+	if err != nil {
+		return nil, nil, err
+	}
+	return &resp, qm, nil
+}
+
 // SchedulerSetConfiguration is used to set the current Scheduler configuration.
 func (op *Operator) SchedulerSetConfiguration(conf *SchedulerConfiguration, q *WriteOptions) (*SchedulerSetConfigurationResponse, *WriteMeta, error) {
 	var out SchedulerSetConfigurationResponse
@@ -258,6 +280,16 @@ func (op *Operator) SchedulerCASConfiguration(conf *SchedulerConfiguration, q *W
 	return &out, wm, nil
 }
 
+// SchedulerSetNumSchedulers is used to set the current number of schedulers
+func (op *Operator) SchedulerSetNumSchedulers(number int, q *WriteOptions) (*GenericResponse, *WriteMeta, error) {
+	var out GenericResponse
+	wm, err := op.c.put("/v1/operator/scheduler/schedulers", SetNumSchedulersRequest{Schedulers: number}, &out, q)
+	if err != nil {
+		return nil, nil, err
+	}
+	return &out, wm, nil
+}
+
 // Snapshot is used to capture a snapshot state of a running cluster.
 // The returned reader that must be consumed fully
 func (op *Operator) Snapshot(q *QueryOptions) (io.ReadCloser, error) {

command/agent/http.go

@@ -501,6 +501,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/system/reconcile/summaries", s.wrap(s.ReconcileJobSummaries))
 
 	s.mux.HandleFunc("/v1/operator/scheduler/configuration", s.wrap(s.OperatorSchedulerConfiguration))
+	s.mux.HandleFunc("/v1/operator/scheduler/schedulers", s.wrap(s.OperatorSchedulers))
 
 	s.mux.HandleFunc("/v1/event/stream", s.wrap(s.EventStream))
 

command/agent/operator_endpoint.go

@@ -342,6 +342,52 @@ func (s *HTTPServer) schedulerUpdateConfig(resp http.ResponseWriter, req *http.R
 	return reply, nil
 }
 
+func (s *HTTPServer) OperatorSchedulers(resp http.ResponseWriter, req *http.Request) (any, error) {
+	// Switch on the method
+	switch req.Method {
+	case http.MethodGet:
+		return s.schedulersGet(resp, req)
+
+	case http.MethodPut, http.MethodPost:
+		return s.schedulersSet(resp, req)
+
+	default:
+		return nil, CodedError(405, ErrInvalidMethod)
+	}
+}
+
+func (s *HTTPServer) schedulersGet(resp http.ResponseWriter, req *http.Request) (any, error) {
+	var args structs.GenericRequest
+	if done := s.parse(resp, req, &args.Region, &args.QueryOptions); done {
+		return nil, nil
+	}
+
+	var reply structs.GetNumSchedulersResponse
+	if err := s.agent.RPC("Operator.GetNumSchedulers", &args, &reply); err != nil {
+		return nil, err
+	}
+	setMeta(resp, &reply.QueryMeta)
+
+	return reply, nil
+}
+
+func (s *HTTPServer) schedulersSet(resp http.ResponseWriter, req *http.Request) (any, error) {
+	var args structs.SetNumSchedulersRequest
+	s.parseWriteRequest(req, &args.WriteRequest)
+
+	var setReq api.SetNumSchedulersRequest
+	if err := decodeBody(req, &setReq); err != nil {
+		return nil, CodedError(http.StatusBadRequest, fmt.Sprintf("Error parsing scheduler config: %v", err))
+	}
+	args.Schedulers = setReq.Schedulers
+
+	var reply structs.GenericResponse
+	if err := s.agent.RPC("Operator.SetNumSchedulers", &args, &reply); err != nil {
+		return nil, err
+	}
+	return reply, nil
+}
+
 func (s *HTTPServer) SnapshotRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	switch req.Method {
 	case http.MethodGet:

command/commands.go

@@ -822,11 +822,21 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"operator scheduler get-num-schedulers": func() (cli.Command, error) {
+			return &OperatorSchedulerGetNumSchedulers{
+				Meta: meta,
+			}, nil
+		},
 		"operator scheduler set-config": func() (cli.Command, error) {
 			return &OperatorSchedulerSetConfig{
 				Meta: meta,
 			}, nil
 		},
+		"operator scheduler set-num-schedulers": func() (cli.Command, error) {
+			return &OperatorSchedulerSetNumSchedulers{
+				Meta: meta,
+			}, nil
+		},
 		"operator root": func() (cli.Command, error) {
 			return &OperatorRootCommand{
 				Meta: meta,

command/operator_scheduler_get_schedulers.go

@@ -0,0 +1,116 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/cli"
+	"github.com/hashicorp/nomad/api"
+	"github.com/posener/complete"
+)
+
+var _ cli.Command = &OperatorSchedulerGetNumSchedulers{}
+
+type OperatorSchedulerGetNumSchedulers struct {
+	Meta
+
+	json bool
+	tmpl string
+}
+
+func (o *OperatorSchedulerGetNumSchedulers) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(o.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-json": complete.PredictNothing,
+			"-t":    complete.PredictAnything,
+		},
+	)
+}
+
+func (o *OperatorSchedulerGetNumSchedulers) AutocompleteArgs() complete.Predictor {
+	return complete.PredictNothing
+}
+
+func (o *OperatorSchedulerGetNumSchedulers) Name() string {
+	return "operator scheduler get-num-schedulers"
+}
+
+func (o *OperatorSchedulerGetNumSchedulers) Run(args []string) int {
+
+	flags := o.Meta.FlagSet("get-num-schedulers", FlagSetClient)
+	flags.BoolVar(&o.json, "json", false, "")
+	flags.StringVar(&o.tmpl, "t", "", "")
+	flags.Usage = func() { o.Ui.Output(o.Help()) }
+
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	// Set up a client.
+	client, err := o.Meta.Client()
+	if err != nil {
+		o.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	// Fetch the current configuration.
+	resp, _, err := client.Operator().SchedulerGetNumSchedulers(&api.QueryOptions{AllowStale: true})
+	if err != nil {
+		o.Ui.Error(fmt.Sprintf("Error querying number of schedulers: %s", err))
+		return 1
+	}
+
+	// If the user has specified to output the scheduler config as JSON or
+	// using a template, perform this action for the entire object and exit the
+	// command.
+	if o.json || len(o.tmpl) > 0 {
+		out, err := Format(o.json, o.tmpl, resp)
+		if err != nil {
+			o.Ui.Error(err.Error())
+			return 1
+		}
+		o.Ui.Output(out)
+		return 0
+	}
+
+	numSched := resp.Schedulers
+
+	// Output the information.
+	o.Ui.Output(formatKV([]string{
+		fmt.Sprintf("Server address|???"),
+		fmt.Sprintf("Number of schedulers|%d", numSched),
+	}))
+	return 0
+}
+
+func (o *OperatorSchedulerGetNumSchedulers) Synopsis() string {
+	return "Display the number of schedulers for the server"
+}
+
+func (o *OperatorSchedulerGetNumSchedulers) Help() string {
+	helpText := `
+Usage: nomad operator scheduler get-schedulers [options]
+
+  Displays the number of schedulers per server.
+
+  If ACLs are enabled, this command requires a token with the 'operator:read'
+  capability.
+
+General Options:
+
+  ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + `
+
+Scheduler Get Config Options:
+
+  -json
+    Output the schedulers in its JSON format.
+
+  -t
+    Format and display the schedulers using a Go template.
+`
+
+	return strings.TrimSpace(helpText)
+}

command/operator_scheduler_set_schedulers.go

@@ -0,0 +1,98 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package command
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/hashicorp/cli"
+	"github.com/posener/complete"
+)
+
+// Ensure OperatorSchedulerSetConfig satisfies the cli.Command interface.
+var _ cli.Command = &OperatorSchedulerSetNumSchedulers{}
+
+type OperatorSchedulerSetNumSchedulers struct {
+	Meta
+}
+
+func (o *OperatorSchedulerSetNumSchedulers) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(o.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-json": complete.PredictNothing,
+		},
+	)
+}
+
+func (o *OperatorSchedulerSetNumSchedulers) AutocompleteArgs() complete.Predictor {
+	return complete.PredictNothing
+}
+
+func (o *OperatorSchedulerSetNumSchedulers) Name() string {
+	return "operator scheduler set-num-schedulers"
+}
+
+func (o *OperatorSchedulerSetNumSchedulers) Run(args []string) int {
+
+	var jsonInput bool
+	flags := o.Meta.FlagSet("set-num-schedulers", FlagSetClient)
+	flags.Usage = func() { o.Ui.Output(o.Help()) }
+	flags.BoolVar(&jsonInput, "json", false, "")
+
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	// Check that we get exactly one argument
+	args = flags.Args()
+	if l := len(args); l != 1 {
+		o.Ui.Error("This command takes one argument: <number>")
+		o.Ui.Error(commandErrorText(o))
+		return 1
+	}
+
+	number, err := strconv.Atoi(args[0])
+	if err != nil {
+		o.Ui.Error(fmt.Sprintf("Invalid number of schedulers to set: %s", args[0]))
+		return 1
+	}
+
+	// Set up a client.
+	client, err := o.Meta.Client()
+	if err != nil {
+		o.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	o.Ui.Info(fmt.Sprintf("Setting number of schedulers to %d...", number))
+	_, _, err = client.Operator().SchedulerSetNumSchedulers(number, nil)
+	if err != nil {
+		o.Ui.Error(fmt.Sprintf("Error setting number of schedulers: %s", err))
+		return 1
+	}
+	o.Ui.Output("Number of schedulers updated")
+	return 0
+}
+
+func (o *OperatorSchedulerSetNumSchedulers) Synopsis() string {
+	return "Modify the current number of schedulers"
+}
+
+func (o *OperatorSchedulerSetNumSchedulers) Help() string {
+	helpText := `
+Usage: nomad operator scheduler set-num-schedulers [options]
+
+  Sets the number of schedulers.
+
+  If ACLs are enabled, this command requires a token with the 'operator:write'
+  capability.
+
+General Options:
+
+  ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + `
+`
+	return strings.TrimSpace(helpText)
+}