{ "accu_init": { "ck_env": [ "--env.CK_LOADGEN_DATASET_SIZE=<<>>" ], "deps": [ "--dep_add_tags.compiler=<<>>", "--dep_add_tags.python=v3", "--dep_add_tags.mlperf-inference-src=r0.7", "--dep_add_tags.weights=<<>>", "--dep_add_tags.edgetpu-lib=<<>>", "--dep_add_tags.dataset=dataset,object-detection,preprocessed,full,using-<<>>" ], "record_tags": [ "mlperf", "division.<<>>", "task.object-detection", "platform.<<>>", "inference_engine.<<>>", "inference_engine_version.<<>>", "inference_engine_backend.<<>>", "scenario.<<>>", "mode.<<>>", "workload.<<>>", "<<>>" ], "record_uoa_parts": [ "mlperf", "<<>>", "<<>>", "<<>>", "<<>>", "<<>>", "<<>>", "<<>>", "<<>>", "<<>><<>>" ] }, "build_map": { "buffer_size": { "###": { "ck_env": "--env.CK_LOADGEN_BUFFER_SIZE=###", "record_tags": "buffer_size.###", "record_uoa_parts": "buffer_size.###" } }, "compile": { "yes": { "shell_wrapped_cmd": "<<>>" } }, "library": { "tflite-v1.13.1": { "deps": "--dep_add_tags.library=tflite,v1.13.1", "program_name": "object-detection-tflite-loadgen", "inference_engine": "tflite", "inference_engine_version": "v1.13.1", "inference_engine_backend": "eigen" }, "tflite-v2.2.0-ruy": { "deps": "--dep_add_tags.library=tflite,v2.2.0", "program_name": "object-detection-tflite-loadgen", "inference_engine": "tflite", "inference_engine_version": "v2.2.0", "inference_engine_backend": "ruy" }, "tflite-v2.3": { "deps": "--dep_add_tags.library=tflite,v2.3.90", "program_name": "object-detection-tflite-loadgen", "inference_engine": "tflite", "inference_engine_version": "v2.3.0", "inference_engine_backend": "eigen" }, "tflite-v2.3-ruy": { "deps": "--dep_add_tags.library=tflite,v2.3.90,with.ruy", "program_name": "object-detection-tflite-loadgen", "inference_engine": "tflite", "inference_engine_version": "v2.3.0", "inference_engine_backend": "ruy" }, "tflite-v2.3-xnnpack": { "deps": "--dep_add_tags.library=tflite,v2.3.90,with.xnnpack", "program_name": "object-detection-tflite-loadgen", "inference_engine": "tflite", "inference_engine_version": "v2.3.0", "inference_engine_backend": "xnnpack" }, "tflite-edgetpu": { "ck_env": "--env.USE_EDGETPU=1", "deps": "--dep_add_tags.library=tflite,for.coral", "program_name": "object-detection-tflite-loadgen", "inference_engine": "tflite", "inference_engine_version": "for.coral", "inference_engine_backend": "edgetpu" } }, "mode": { "accuracy": { "ck_env": "--env.CK_LOADGEN_MODE=AccuracyOnly", "mode_extension": "-dataset_size.<<>>-<<>>", "output_file": "accuracy.txt", "recommended_dataset_size": "50000" }, "performance": { "ck_env": "--env.CK_LOADGEN_MODE=PerformanceOnly", "mode_extension": "<<>>", "output_file": "mlperf_log_summary.txt", "recommended_dataset_size": "1024" } }, "model": { "non-quantized": { "division": "closed", "model_tags": [ "ssd-mobilenet", "###" ] }, "v[12]": { "division": "open", "model_tags": [ "ssd-mobilenet", "###", "edgetpu" ] } }, "model_extra_tags": { "###": { "model_tags": [ "###" ] } }, "edgetpu_lib": { "std": { "edgetpu_tags": [ "std_14.1_arm64" ] }, "max": { "edgetpu_tags": [ "max_14.1_arm64" ] } }, "no_record": { "yes": { "shell_wrapped_cmd": "<<>>" } }, "performance_sample_count_override": { "###": { "ck_env": "--env.CK_LOADGEN_BUFFER_SIZE=###", "record_tags": "buffer_size.###", "record_uoa_parts": "buffer_size.###" } }, "scenario": { "offline": { "ck_env": "--env.CK_LOADGEN_SCENARIO=Offline", "performance_mode_extension": "" }, "singlestream": { "ck_env": "--env.CK_LOADGEN_SCENARIO=SingleStream", "performance_mode_extension": "-target_latency.<<>>" } }, "sut": { "###": { "compiler_tags": "gcc,v8" }, "firefly": { "compiler_tags": "gcc,v7" }, "hikey": { "compiler_tags": "gcc,v7" }, "mac": { "compiler_tags": "llvm,v10" }, "mate10pro": { "compiler_tags": "llvm,v6" }, "rpi4": { "compiler_tags": "gcc,v8" }, "rpi4coral": { "compiler_tags": "gcc,v9" }, "xavier": { "compiler_tags": "gcc,v7" } }, "target_latency": { "###": { "ck_env": "--env.CK_LOADGEN_TARGET_LATENCY=###", "record_tags": "target_latency.###" } }, "target_qps": { "###": { "ck_env": "--env.CK_LOADGEN_TARGET_QPS=###", "record_tags": "target_qps.###" } }, "verbose": { "all": { "print_output_file": "<<>>", "screen_output": "" }, "yes": { "print_output_file": "<<>>" } } }, "cmd_template": "<<>>", "defaults": { "ck_benchmark_program": "ck benchmark <<>> --skip_stat_analysis --process_multi_keys --repetitions=<<>> --record --record_repo=local --record_uoa=<<>> --tags=<<>>", "ck_compile_program": "ck compile program:<<>> <<>>", "ck_run_program": "ck run <<>>", "dataset_size": "<<>>", "division": "open", "model_readable": "<<>>", "preprocessed_tag": "preprocessed_using.<<>>", "preprocessed_using": "pillow", "print_output_file_cmd": "&& cat `ck find program:<<>>`/tmp/<<>>", "record_uoa": "<<>>", "repetitions": "1", "run_or_benchmark_common": "program:<<>> <<>> <<>> <<>>", "screen_output": "--env.CK_SILENT_MODE=YES --skip_print_timers", "shell_wrapped_cmd": "<<>>", "wrapped_ck_benchmark_program": "if [ -n \"`ck list local:experiment:<<>>`\" ]; then echo 'Experiment <<>> exists, skipping' ; else <<>> \"@@@{'meta': {'ck_benchmark_program': '<<>>'}}\" <<>> ; fi", "wrapped_ck_run_program": "<<>> <<>>" } }