31
31
from . import cli_create
32
32
from . import cli_deploy
33
33
from .. import version
34
+ from ..evaluation .gcs_eval_set_results_manager import GcsEvalSetResultsManager
35
+ from ..evaluation .gcs_eval_sets_manager import GcsEvalSetsManager
34
36
from ..evaluation .local_eval_set_results_manager import LocalEvalSetResultsManager
35
37
from ..sessions .in_memory_session_service import InMemorySessionService
36
38
from .cli import run_cli
37
39
from .cli_eval import MISSING_EVAL_DEPENDENCIES_MESSAGE
38
40
from .fast_api import get_fast_api_app
39
41
from .utils import envs
42
+ from .utils import evals
40
43
from .utils import logs
41
44
42
45
LOG_LEVELS = click .Choice (
@@ -282,11 +285,21 @@ def cli_run(
282
285
default = False ,
283
286
help = "Optional. Whether to print detailed results on console or not." ,
284
287
)
288
+ @click .option (
289
+ "--eval_storage_uri" ,
290
+ type = str ,
291
+ help = (
292
+ "Optional. The evals storage URI to store agent evals,"
293
+ " supported URIs: gs://<bucket name>."
294
+ ),
295
+ default = None ,
296
+ )
285
297
def cli_eval (
286
298
agent_module_file_path : str ,
287
- eval_set_file_path : tuple [str ],
299
+ eval_set_file_path : list [str ],
288
300
config_file_path : str ,
289
301
print_detailed_results : bool ,
302
+ eval_storage_uri : Optional [str ] = None ,
290
303
):
291
304
"""Evaluates an agent given the eval sets.
292
305
@@ -338,12 +351,33 @@ def cli_eval(
338
351
root_agent = get_root_agent (agent_module_file_path )
339
352
reset_func = try_get_reset_func (agent_module_file_path )
340
353
354
+ gcs_eval_sets_manager = None
355
+ eval_set_results_manager = None
356
+ if eval_storage_uri :
357
+ gcs_eval_managers = evals .create_gcs_eval_managers_from_uri (
358
+ eval_storage_uri
359
+ )
360
+ gcs_eval_sets_manager = gcs_eval_managers .eval_sets_manager
361
+ eval_set_results_manager = gcs_eval_managers .eval_set_results_manager
362
+ else :
363
+ eval_set_results_manager = LocalEvalSetResultsManager (
364
+ agents_dir = os .path .dirname (agent_module_file_path )
365
+ )
341
366
eval_set_file_path_to_evals = parse_and_get_evals_to_run (eval_set_file_path )
342
367
eval_set_id_to_eval_cases = {}
343
368
344
369
# Read the eval_set files and get the cases.
345
370
for eval_set_file_path , eval_case_ids in eval_set_file_path_to_evals .items ():
346
- eval_set = load_eval_set_from_file (eval_set_file_path , eval_set_file_path )
371
+ if gcs_eval_sets_manager :
372
+ eval_set = gcs_eval_sets_manager ._load_eval_set_from_blob (
373
+ eval_set_file_path
374
+ )
375
+ if not eval_set :
376
+ raise click .ClickException (
377
+ f"Eval set { eval_set_file_path } not found in GCS."
378
+ )
379
+ else :
380
+ eval_set = load_eval_set_from_file (eval_set_file_path , eval_set_file_path )
347
381
eval_cases = eval_set .eval_cases
348
382
349
383
if eval_case_ids :
@@ -378,16 +412,13 @@ async def _collect_eval_results() -> list[EvalCaseResult]:
378
412
raise click .ClickException (MISSING_EVAL_DEPENDENCIES_MESSAGE )
379
413
380
414
# Write eval set results.
381
- local_eval_set_results_manager = LocalEvalSetResultsManager (
382
- agents_dir = os .path .dirname (agent_module_file_path )
383
- )
384
415
eval_set_id_to_eval_results = collections .defaultdict (list )
385
416
for eval_case_result in eval_results :
386
417
eval_set_id = eval_case_result .eval_set_id
387
418
eval_set_id_to_eval_results [eval_set_id ].append (eval_case_result )
388
419
389
420
for eval_set_id , eval_case_results in eval_set_id_to_eval_results .items ():
390
- local_eval_set_results_manager .save_eval_set_result (
421
+ eval_set_results_manager .save_eval_set_result (
391
422
app_name = os .path .basename (agent_module_file_path ),
392
423
eval_set_id = eval_set_id ,
393
424
eval_case_results = eval_case_results ,
@@ -444,6 +475,15 @@ def decorator(func):
444
475
),
445
476
default = None ,
446
477
)
478
+ @click .option (
479
+ "--eval_storage_uri" ,
480
+ type = str ,
481
+ help = (
482
+ "Optional. The evals storage URI to store agent evals,"
483
+ " supported URIs: gs://<bucket name>."
484
+ ),
485
+ default = None ,
486
+ )
447
487
@click .option (
448
488
"--memory_service_uri" ,
449
489
type = str ,
@@ -564,6 +604,7 @@ def wrapper(*args, **kwargs):
564
604
)
565
605
def cli_web (
566
606
agents_dir : str ,
607
+ eval_storage_uri : Optional [str ] = None ,
567
608
log_level : str = "INFO" ,
568
609
allow_origins : Optional [list [str ]] = None ,
569
610
host : str = "127.0.0.1" ,
@@ -616,6 +657,7 @@ async def _lifespan(app: FastAPI):
616
657
session_service_uri = session_service_uri ,
617
658
artifact_service_uri = artifact_service_uri ,
618
659
memory_service_uri = memory_service_uri ,
660
+ eval_storage_uri = eval_storage_uri ,
619
661
allow_origins = allow_origins ,
620
662
web = True ,
621
663
trace_to_cloud = trace_to_cloud ,
@@ -654,6 +696,7 @@ async def _lifespan(app: FastAPI):
654
696
)
655
697
def cli_api_server (
656
698
agents_dir : str ,
699
+ eval_storage_uri : Optional [str ] = None ,
657
700
log_level : str = "INFO" ,
658
701
allow_origins : Optional [list [str ]] = None ,
659
702
host : str = "127.0.0.1" ,
@@ -685,6 +728,7 @@ def cli_api_server(
685
728
session_service_uri = session_service_uri ,
686
729
artifact_service_uri = artifact_service_uri ,
687
730
memory_service_uri = memory_service_uri ,
731
+ eval_storage_uri = eval_storage_uri ,
688
732
allow_origins = allow_origins ,
689
733
web = False ,
690
734
trace_to_cloud = trace_to_cloud ,
@@ -771,6 +815,15 @@ def cli_api_server(
771
815
" version in the dev environment)"
772
816
),
773
817
)
818
+ @click .option (
819
+ "--eval_storage_uri" ,
820
+ type = str ,
821
+ help = (
822
+ "Optional. The evals storage URI to store agent evals,"
823
+ " supported URIs: gs://<bucket name>."
824
+ ),
825
+ default = None ,
826
+ )
774
827
@adk_services_options ()
775
828
@deprecated_adk_services_options ()
776
829
@click .argument (
@@ -797,6 +850,7 @@ def cli_deploy_cloud_run(
797
850
session_service_uri : Optional [str ] = None ,
798
851
artifact_service_uri : Optional [str ] = None ,
799
852
memory_service_uri : Optional [str ] = None ,
853
+ eval_storage_uri : Optional [str ] = None ,
800
854
session_db_url : Optional [str ] = None , # Deprecated
801
855
artifact_storage_uri : Optional [str ] = None , # Deprecated
802
856
):
0 commit comments