{
  "accu_init": {
    "ck_env": [
      "--env.CK_LOADGEN_DATASET_SIZE=<<<dataset_size>>>",
      "--env.CK_VERBOSE=<<<verbosity>>>"
    ],
    "record_tags": [
      "mlperf",
      "division.<<<division>>>",
      "task.<<<task>>>",
      "inference_engine.<<<inference_engine>>>",
      "inference_engine_version.<<<inference_engine_version>>>",
      "platform.<<<sut>>>",
      "scenario.<<<scenario>>>",
      "mode.<<<mode>>>",
      "workload.<<<model>>>"
    ],
    "record_uoa_parts": [
      "mlperf",
      "<<<division>>>",
      "<<<sut>>>",
      "<<<inference_engine>>>",
      "<<<inference_engine_version>>>",
      "<<<model>>>",
      "<<<scenario>>>"
    ]
  },
  "build_map": {
    "mode": {
      "accuracy": {
        "ck_env": "--cmd_key=accuracy",
        "output_file": "accuracy.txt",
        "recommended_dataset_size": "<<<full_dataset_size>>>",
        "recommended_verbosity": "1"
      },
      "performance": {
        "ck_env": "--cmd_key=performance",
        "output_file": "mlperf_log_summary.txt",
        "recommended_dataset_size": "<<<performance_dataset_size>>>",
        "recommended_verbosity": "0"
      }
    },
    "model": {
      "rnnt": {
        "full_dataset_size": "2513",
        "performance_dataset_size": "2513",
        "program_name": "speech-recognition-pytorch-loadgen",
        "task": "speech-recognition"
      }
    },
    "no_record": {
      "yes": {
        "shell_wrapped_cmd": "<<<wrapped_ck_run_program>>>"
      }
    },
    "scenario": {
      "multistream": {
        "ck_env": "--env.CK_LOADGEN_SCENARIO=MultiStream"
      },
      "offline": {
        "ck_env": "--env.CK_LOADGEN_SCENARIO=Offline"
      },
      "singlestream": {
        "ck_env": "--env.CK_LOADGEN_SCENARIO=SingleStream"
      },
      "server": {
        "ck_env": "--env.CK_LOADGEN_SCENARIO=Server"
      }
    },
    "sut": {
      "aws-g4dn.4xlarge": {
        "compiler_tags": "gcc,v7"
      },
      "dellemc_r640xd6248r": {
        "compiler_tags": "gcc,v7"
      },
      "dellemc_r740xd6248": {
        "compiler_tags": "gcc,v7"
      }
    },
    "batch_size": {
      "###": {
        "ck_env": "--env.CK_BATCH_SIZE=###",
        "record_tags": "batch_size.###"
      }
    },
    "buffer_size": {
      "###": {
        "ck_env": "--env.CK_LOADGEN_BUFFER_SIZE=###",
        "record_tags": "buffer_size.###",
        "record_uoa_parts": "buffer_size.###"
      }
    },
    "performance_sample_count_override": {
      "###": {
        "ck_env": "--env.CK_LOADGEN_BUFFER_SIZE=###",
        "record_tags": "buffer_size.###",
        "record_uoa_parts": "buffer_size.###"
      }
    },
    "max_query_count": {
      "###": {
        "ck_env": "--env.CK_LOADGEN_MAX_QUERY_COUNT=###",
        "record_tags": "max_query_count.###",
        "record_uoa_parts": "max_query_count.###"
      }
    },
    "target_latency": {
      "###": {
        "ck_env": "--env.CK_LOADGEN_TARGET_LATENCY=###",
        "record_tags": "target_latency.###"
      }
    },
    "target_qps": {
      "###": {
        "ck_env": "--env.CK_LOADGEN_TARGET_QPS=###",
        "record_tags": "target_qps.###",
        "record_uoa_parts": "target_qps.###"
      }
    },
    "no_record": {
      "yes": {
        "shell_wrapped_cmd": "<<<wrapped_ck_run_program>>>"
      }
    },
    "verbosity": {
      "###": {
        "ck_env": "--env.CK_VERBOSE=###"
      }
    },    "verbose": {
      "all": {
        "print_output_file": "<<<print_output_file_cmd>>>",
        "screen_output": ""
      },
      "yes": {
        "print_output_file": "<<<print_output_file_cmd>>>"
      }
    },
    "plugin_pre": {
      "pytorch": {
        "ck_env": ["--env.CK_RNNT_PRE_BACKEND=pytorch", "--env.CK_RNNT_PRE=orig"]
      },
      "pytorch_quant": {
        "ck_env": ["--env.CK_RNNT_PRE_BACKEND=pytorch", "--env.CK_RNNT_PRE=orig-quant"]
      },
      "onnx": {
        "ck_env": ["--env.CK_RNNT_PRE_BACKEND=onnx", "--env.CK_RNNT_PRE=orig"]
      }
    },
    "plugin_post": {
      "pytorch": {
        "ck_env": ["--env.CK_RNNT_POST_BACKEND=pytorch", "--env.CK_RNNT_POST=orig"]
      },
      "pytorch_quant": {
        "ck_env": ["--env.CK_RNNT_POST_BACKEND=pytorch", "--env.CK_RNNT_POST=orig-quant"]
      },
      "onnx": {
        "ck_env": ["--env.CK_RNNT_POST_BACKEND=onnx", "--env.CK_RNNT_POST=orig"]
      }
    },
    "plugin_dec": {
      "pytorch": {
        "ck_env": ["--env.CK_RNNT_DEC_BACKEND=pytorch", "--env.CK_RNNT_DEC=orig"]
      },
      "pytorch_quant": {
        "ck_env": ["--env.CK_RNNT_DEC_BACKEND=pytorch", "--env.CK_RNNT_DEC=orig-quant"]
      },
      "onnx": {
        "ck_env": ["--env.CK_RNNT_DEC_BACKEND=onnx", "--env.CK_RNNT_DEC=orig"]
      }
    }
  },
  "cmd_template": "<<<shell_wrapped_cmd>>>",
  "defaults": {
    "ck_benchmark_program": "ck benchmark <<<run_or_benchmark_common>>> --skip_stat_analysis --process_multi_keys --repetitions=<<<repetitions>>> --record --record_repo=local --record_uoa=<<<record_uoa>>> --tags=<<<record_tags,>>>",
    "ck_run_program": "ck run <<<run_or_benchmark_common>>>",
    "dataset_size": "<<<recommended_dataset_size>>>",
    "division": "closed",
    "inference_engine": "pytorch",
    "inference_engine_version": "v1.15.1",
    "print_output_file_cmd": "&& cat `ck find program:<<<program_name>>>`/tmp/<<<output_file>>>",
    "record_uoa": "<<<record_uoa_parts->>>",
    "repetitions": "1",
    "run_or_benchmark_common": "program:<<<program_name>>> <<<screen_output>>> <<<?deps >>> <<<?ck_env >>>",
    "screen_output": "--env.CK_SILENT_MODE=YES --skip_print_timers",
    "shell_wrapped_cmd": "<<<wrapped_ck_benchmark_program>>>",
    "verbosity": "<<<recommended_verbosity>>>",
    "wrapped_ck_benchmark_program": "if [ -n \"`ck list local:experiment:<<<record_uoa>>>`\" ]; then echo 'Experiment <<<record_uoa>>> exists, skipping' ; else <<<ck_benchmark_program>>> \"@@@{'meta': {'ck_benchmark_program': '<<<ck_benchmark_program>>>'}}\" <<<?print_output_file>>> ; fi",
    "wrapped_ck_run_program": "<<<ck_run_program>>> <<<?print_output_file>>>"
  }
}