Skip to content

Commit 15bfa0b

Browse files
irar2irar2
authored
Enable configuration of various parameters in tools (#100)
* Enable configuration of various parameters in tools Signed-off-by: Ira <IRAR@il.ibm.com> * Review comments Signed-off-by: Ira <IRAR@il.ibm.com> --------- Signed-off-by: Ira <IRAR@il.ibm.com>
1 parent 477508d commit 15bfa0b

File tree

7 files changed

+343
-91
lines changed

7 files changed

+343
-91
lines changed

‎README.md

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,15 @@ For more details see the <a href="https://docs.vllm.ai/en/stable/getting_started
101101
- `inter-token-latency`: the time to 'generate' each additional token (in milliseconds), optional, by default zero
102102
- `kv-cache-transfer-latency`: time for KV-cache transfer from a remote vLLM (in milliseconds), by default zero. Usually much shorter than `time-to-first-token`
103103
- `seed`: random seed for operations (if not set, current Unix time in nanoseconds is used)
104-
104+
- `max-tool-call-integer-param`: the maximum possible value of integer parameters in a tool call, optional, defaults to 100
105+
- `min-tool-call-integer-param`: the minimum possible value of integer parameters in a tool call, optional, defaults to 0
106+
- `max-tool-call-number-param`: the maximum possible value of number (float) parameters in a tool call, optional, defaults to 100
107+
- `min-tool-call-number-param`: the minimum possible value of number (float) parameters in a tool call, optional, defaults to 0
108+
- `max-tool-call-array-param-length`: the maximum possible length of array parameters in a tool call, optional, defaults to 5
109+
- `min-tool-call-array-param-length`: the minimum possible length of array parameters in a tool call, optional, defaults to 1
110+
- `tool-call-not-required-param-probability`: the probability to add a parameter, that is not required, in a tool call, optional, defaults to 50
111+
- `object-tool-call-not-required-field-probability`: the probability to add a field, that is not required, in an object in a tool call, optional, defaults to 50
112+
105113
In addition, as we are using klog, the following parameters are available:
106114
- `add_dir_header`: if true, adds the file directory to the header of the log messages
107115
- `alsologtostderr`: log to standard error as well as files (no effect when -logtostderr=true)

‎pkg/llm-d-inference-sim/config.go

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,33 @@ type configuration struct {
6060
Mode string `yaml:"mode"`
6161
// Seed defines random seed for operations
6262
Seed int64 `yaml:"seed"`
63+
64+
// MaxToolCallIntegerParam defines the maximum possible value of integer parameters in a tool call,
65+
// optional, defaults to 100
66+
MaxToolCallIntegerParam int `yaml:"max-tool-call-integer-param"`
67+
// MinToolCallIntegerParam defines the minimum possible value of integer parameters in a tool call,
68+
// optional, defaults to 0
69+
MinToolCallIntegerParam int `yaml:"min-tool-call-integer-param"`
70+
// MaxToolCallNumberParam defines the maximum possible value of number (float) parameters in a tool call,
71+
// optional, defaults to 100
72+
MaxToolCallNumberParam float64 `yaml:"max-tool-call-number-param"`
73+
// MinToolCallNumberParam defines the minimum possible value of number (float) parameters in a tool call,
74+
// optional, defaults to 0
75+
MinToolCallNumberParam float64 `yaml:"min-tool-call-number-param"`
76+
77+
// MaxToolCallArrayParamLength defines the maximum possible length of array parameters in a tool call,
78+
// optional, defaults to 5
79+
MaxToolCallArrayParamLength int `yaml:"max-tool-call-array-param-length"`
80+
// MinToolCallArrayParamLength defines the minimum possible length of array parameters in a tool call,
81+
// optional, defaults to 1
82+
MinToolCallArrayParamLength int `yaml:"min-tool-call-array-param-length"`
83+
84+
// ToolCallNotRequiredParamProbability is the probability to add a parameter, that is not required,
85+
// in a tool call, optional, defaults to 50
86+
ToolCallNotRequiredParamProbability int `yaml:"tool-call-not-required-param-probability"`
87+
// ObjectToolCallNotRequiredParamProbability is the probability to add a field, that is not required,
88+
// in an object in a tool call, optional, defaults to 50
89+
ObjectToolCallNotRequiredParamProbability int `yaml:"object-tool-call-not-required-field-probability"`
6390
}
6491

6592
type loraModule struct {
@@ -103,12 +130,18 @@ func (c *configuration) unmarshalLoras() error {
103130

104131
func newConfig() *configuration {
105132
return &configuration{
106-
Port: vLLMDefaultPort,
107-
MaxLoras: 1,
108-
MaxNumSeqs: 5,
109-
MaxModelLen: 1024,
110-
Mode: modeRandom,
111-
Seed: time.Now().UnixNano(),
133+
Port: vLLMDefaultPort,
134+
MaxLoras: 1,
135+
MaxNumSeqs: 5,
136+
MaxModelLen: 1024,
137+
Mode: modeRandom,
138+
Seed: time.Now().UnixNano(),
139+
MaxToolCallIntegerParam: 100,
140+
MaxToolCallNumberParam: 100,
141+
MaxToolCallArrayParamLength: 5,
142+
MinToolCallArrayParamLength: 1,
143+
ToolCallNotRequiredParamProbability: 50,
144+
ObjectToolCallNotRequiredParamProbability: 50,
112145
}
113146
}
114147

@@ -174,5 +207,23 @@ func (c *configuration) validate() error {
174207
}
175208
}
176209

210+
if c.MaxToolCallIntegerParam < c.MinToolCallIntegerParam {
211+
return errors.New("MaxToolCallIntegerParam cannot be less than MinToolCallIntegerParam")
212+
}
213+
if c.MaxToolCallNumberParam < c.MinToolCallNumberParam {
214+
return errors.New("MaxToolCallNumberParam cannot be less than MinToolCallNumberParam")
215+
}
216+
if c.MaxToolCallArrayParamLength < c.MinToolCallArrayParamLength {
217+
return errors.New("MaxToolCallArrayParamLength cannot be less than MinToolCallArrayParamLength")
218+
}
219+
if c.MinToolCallArrayParamLength < 0 {
220+
return errors.New("MinToolCallArrayParamLength cannot be negative")
221+
}
222+
if c.ToolCallNotRequiredParamProbability < 0 || c.ToolCallNotRequiredParamProbability > 100 {
223+
return errors.New("ToolCallNotRequiredParamProbability should be between 0 and 100")
224+
}
225+
if c.ObjectToolCallNotRequiredParamProbability < 0 || c.ObjectToolCallNotRequiredParamProbability > 100 {
226+
return errors.New("ObjectToolCallNotRequiredParamProbability should be between 0 and 100")
227+
}
177228
return nil
178229
}

‎pkg/llm-d-inference-sim/config_test.go

Lines changed: 71 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -192,70 +192,80 @@ var _ = Describe("Simulator configuration", func() {
192192
}
193193
tests = append(tests, test)
194194

195-
// Invalid configurations
196-
test = testCase{
197-
name: "invalid model",
198-
args: []string{"cmd", "--model", "", "--config", "../../manifests/config.yaml"},
199-
}
200-
tests = append(tests, test)
201-
202-
test = testCase{
203-
name: "invalid port",
204-
args: []string{"cmd", "--port", "-50", "--config", "../../manifests/config.yaml"},
205-
}
206-
tests = append(tests, test)
207-
208-
test = testCase{
209-
name: "invalid max-loras",
210-
args: []string{"cmd", "--max-loras", "15", "--config", "../../manifests/config.yaml"},
195+
for _, test := range tests {
196+
When(test.name, func() {
197+
It("should create correct configuration", func() {
198+
config, err := createSimConfig(test.args)
199+
Expect(err).NotTo(HaveOccurred())
200+
Expect(config).To(Equal(test.expectedConfig))
201+
})
202+
})
211203
}
212-
tests = append(tests, test)
213204

214-
test = testCase{
215-
name: "invalid mode",
216-
args: []string{"cmd", "--mode", "hello", "--config", "../../manifests/config.yaml"},
217-
}
218-
tests = append(tests, test)
219-
220-
test = testCase{
221-
name: "invalid lora",
222-
args: []string{"cmd", "--config", "../../manifests/config.yaml",
223-
"--lora-modules", "[{\"path\":\"/path/to/lora15\"}]"},
205+
// Invalid configurations
206+
invalidTests := []testCase{
207+
{
208+
name: "invalid model",
209+
args: []string{"cmd", "--model", "", "--config", "../../manifests/config.yaml"},
210+
},
211+
{
212+
name: "invalid port",
213+
args: []string{"cmd", "--port", "-50", "--config", "../../manifests/config.yaml"},
214+
},
215+
{
216+
name: "invalid max-loras",
217+
args: []string{"cmd", "--max-loras", "15", "--config", "../../manifests/config.yaml"},
218+
},
219+
{
220+
name: "invalid mode",
221+
args: []string{"cmd", "--mode", "hello", "--config", "../../manifests/config.yaml"},
222+
},
223+
{
224+
name: "invalid lora",
225+
args: []string{"cmd", "--config", "../../manifests/config.yaml",
226+
"--lora-modules", "[{\"path\":\"/path/to/lora15\"}]"},
227+
},
228+
{
229+
name: "invalid max-model-len",
230+
args: []string{"cmd", "--max-model-len", "0", "--config", "../../manifests/config.yaml"},
231+
},
232+
{
233+
name: "invalid tool-call-not-required-param-probability",
234+
args: []string{"cmd", "--tool-call-not-required-param-probability", "-10", "--config", "../../manifests/config.yaml"},
235+
},
236+
{
237+
name: "invalid max-tool-call-number-param",
238+
args: []string{"cmd", "--max-tool-call-number-param", "-10", "--min-tool-call-number-param", "0",
239+
"--config", "../../manifests/config.yaml"},
240+
},
241+
{
242+
name: "invalid max-tool-call-integer-param",
243+
args: []string{"cmd", "--max-tool-call-integer-param", "-10", "--min-tool-call-integer-param", "0",
244+
"--config", "../../manifests/config.yaml"},
245+
},
246+
{
247+
name: "invalid max-tool-call-array-param-length",
248+
args: []string{"cmd", "--max-tool-call-array-param-length", "-10", "--min-tool-call-array-param-length", "0",
249+
"--config", "../../manifests/config.yaml"},
250+
},
251+
{
252+
name: "invalid tool-call-not-required-param-probability",
253+
args: []string{"cmd", "--tool-call-not-required-param-probability", "-10",
254+
"--config", "../../manifests/config.yaml"},
255+
},
256+
{
257+
name: "invalid object-tool-call-not-required-field-probability",
258+
args: []string{"cmd", "--object-tool-call-not-required-field-probability", "1210",
259+
"--config", "../../manifests/config.yaml"},
260+
},
224261
}
225-
tests = append(tests, test)
226262

227-
test = testCase{
228-
name: "invalid max-model-len",
229-
args: []string{"cmd", "--max-model-len", "0", "--config", "../../manifests/config.yaml"},
263+
for _, test := range invalidTests {
264+
When(test.name, func() {
265+
It("should fail for invalid configuration", func() {
266+
_, err := createSimConfig(test.args)
267+
Expect(err).To(HaveOccurred())
268+
})
269+
})
230270
}
231-
tests = append(tests, test)
232-
233-
DescribeTable("check configurations",
234-
func(args []string, expectedConfig *configuration) {
235-
config, err := createSimConfig(args)
236-
Expect(err).NotTo(HaveOccurred())
237-
Expect(config).To(Equal(expectedConfig))
238-
},
239-
Entry(tests[0].name, tests[0].args, tests[0].expectedConfig),
240-
Entry(tests[1].name, tests[1].args, tests[1].expectedConfig),
241-
Entry(tests[2].name, tests[2].args, tests[2].expectedConfig),
242-
Entry(tests[3].name, tests[3].args, tests[3].expectedConfig),
243-
Entry(tests[4].name, tests[4].args, tests[4].expectedConfig),
244-
Entry(tests[5].name, tests[5].args, tests[5].expectedConfig),
245-
Entry(tests[6].name, tests[6].args, tests[6].expectedConfig),
246-
Entry(tests[7].name, tests[7].args, tests[7].expectedConfig),
247-
)
248-
249-
DescribeTable("invalid configurations",
250-
func(args []string) {
251-
_, err := createSimConfig(args)
252-
Expect(err).To(HaveOccurred())
253-
},
254-
Entry(tests[8].name, tests[8].args),
255-
Entry(tests[9].name, tests[9].args),
256-
Entry(tests[10].name, tests[10].args),
257-
Entry(tests[11].name, tests[11].args),
258-
Entry(tests[12].name, tests[12].args),
259-
Entry(tests[13].name, tests[13].args),
260-
)
261271
})

‎pkg/llm-d-inference-sim/simulator.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,15 @@ func (s *VllmSimulator) parseCommandParamsAndLoadConfig() error {
160160
f.IntVar(&config.KVCacheTransferLatency, "kv-cache-transfer-latency", config.KVCacheTransferLatency, "Time for KV-cache transfer from a remote vLLM (in milliseconds)")
161161
f.Int64Var(&config.Seed, "seed", config.Seed, "Random seed for operations (if not set, current Unix time in nanoseconds is used)")
162162

163+
f.IntVar(&config.MaxToolCallIntegerParam, "max-tool-call-integer-param", config.MaxToolCallIntegerParam, "Maximum possible value of integer parameters in a tool call")
164+
f.IntVar(&config.MinToolCallIntegerParam, "min-tool-call-integer-param", config.MinToolCallIntegerParam, "Minimum possible value of integer parameters in a tool call")
165+
f.Float64Var(&config.MaxToolCallNumberParam, "max-tool-call-number-param", config.MaxToolCallNumberParam, "Maximum possible value of number (float) parameters in a tool call")
166+
f.Float64Var(&config.MinToolCallNumberParam, "min-tool-call-number-param", config.MinToolCallNumberParam, "Minimum possible value of number (float) parameters in a tool call")
167+
f.IntVar(&config.MaxToolCallArrayParamLength, "max-tool-call-array-param-length", config.MaxToolCallArrayParamLength, "Maximum possible length of array parameters in a tool call")
168+
f.IntVar(&config.MinToolCallArrayParamLength, "min-tool-call-array-param-length", config.MinToolCallArrayParamLength, "Minimum possible length of array parameters in a tool call")
169+
f.IntVar(&config.ToolCallNotRequiredParamProbability, "tool-call-not-required-param-probability", config.ToolCallNotRequiredParamProbability, "Probability to add a parameter, that is not required, in a tool call")
170+
f.IntVar(&config.ObjectToolCallNotRequiredParamProbability, "object-tool-call-not-required-field-probability", config.ObjectToolCallNotRequiredParamProbability, "Probability to add a field, that is not required, in an object in a tool call")
171+
163172
// These values were manually parsed above in getParamValueFromArgs, we leave this in order to get these flags in --help
164173
var dummyString string
165174
f.StringVar(&dummyString, "config", "", "The path to a yaml configuration file. The command line values overwrite the configuration file values")
@@ -462,7 +471,7 @@ func (s *VllmSimulator) reqProcessingWorker(ctx context.Context, id int) {
462471
req.getToolChoice() != toolChoiceNone &&
463472
req.getTools() != nil {
464473
toolCalls, finishReason, completionTokens, err =
465-
createToolCalls(req.getTools(), req.getToolChoice())
474+
createToolCalls(req.getTools(), req.getToolChoice(), s.config)
466475
}
467476
if toolCalls == nil && err == nil {
468477
// Either no tool calls were defined, or we randomly chose not to create tool calls,

0 commit comments

Comments
 (0)