[ { "50.00 percentile latency (ns)": 130851575, "90.00 percentile latency (ns)": 231996729, "90th percentile latency (ns)": 231996729, "95.00 percentile latency (ns)": 246342099, "97.00 percentile latency (ns)": 256104655, "99.00 percentile latency (ns)": 279368826, "99.90 percentile latency (ns)": 344096220, "Max latency (ns)": 377036316, "Mean latency (ns)": 139409423, "Min duration satisfied": "Yes", "Min latency (ns)": 33388642, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 7.17, "QPS w/o loadgen overhead": 7.17, "Result is": "VALID", "SUT name": "GNMT SERVER", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier", "accelerator_on-chip_memories": "1MB (128KB/SM) L1 + 512KB L2 + 4MB (DLA)", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.90th_percentile_latency_ms": 231.996729, "characteristics.90th_percentile_latency_ns": 231996729.0, "characteristics.90th_percentile_latency_s": 0.231996729, "characteristics.90th_percentile_latency_us": 231996.729, "characteristics.blue": 23.9, "ck_system": "Xavier", "ck_used": true, "cooling": "", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "open", "formal_model": "gnmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.3 DP, TensorRT 6.0, cuDNN 7.6.3, CUDA 10.0, cub 1.8.0", "host_memory_capacity": "16 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.5 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.5, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/open/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/open/NVIDIA/results/Xavier", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "pycuda 2019.1, pytorch 1.1, torchvision 0.2.2.post3; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 1, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "target_latency (ns)": 0, "target_qps": 4.27899, "task": "NMT", "task2": "nmt", "total_cores": 120, "uid": "7a309aba4735bf32", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 13944258, "90.00 percentile latency (ns)": 24429173, "90th percentile latency (ns)": 24429173, "95.00 percentile latency (ns)": 25838066, "97.00 percentile latency (ns)": 26963511, "99.00 percentile latency (ns)": 30724580, "99.90 percentile latency (ns)": 39671992, "Max latency (ns)": 44055983, "Mean latency (ns)": 14892858, "Min duration satisfied": "Yes", "Min latency (ns)": 2456800, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 66.99, "QPS w/o loadgen overhead": 67.15, "Result is": "VALID", "SUT name": "GNMT SERVER", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA TITAN RTX", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.90th_percentile_latency_ms": 24.429173, "characteristics.90th_percentile_latency_ns": 24429173.0, "characteristics.90th_percentile_latency_s": 0.024429173, "characteristics.90th_percentile_latency_us": 24429.173, "characteristics.blue": 23.7, "ck_system": "TitanRTXx4", "ck_used": false, "cooling": "watercooled", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "closed", "formal_model": "nmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 6.0, CUDA 10.1, cuDNN 7.6.3, libjemalloc2, cub 1.8.0, tensorrt-laboratory mlperf branch", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 24, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) 8268", "host_processors_per_node": 2, "host_storage_capacity": "3.84 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.5, "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/results/TitanRTXx4", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.3", "other_software_stack": "docker 18.09.2, python 3.6.8,gcc 5.5.0,onnx 1.3.0, tensorflow 1.13.1, pytorch 1.1.0, torchvision 0.3.0, pycuda 2019.1, sacrebleu 1.3.3, SimpleJSON, OpenCV 4.1.1", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 1, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/TitanRTXx4", "system_name": "SCAN 3XS DBP T496X2 Fluid", "target_latency (ns)": 0, "target_qps": 36.5894, "task": "NMT", "task2": "nmt", "total_cores": 48, "uid": "767bb447b77eeb08", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 23433741, "90.00 percentile latency (ns)": 41322309, "90th percentile latency (ns)": 41322309, "95.00 percentile latency (ns)": 44052014, "97.00 percentile latency (ns)": 45797965, "99.00 percentile latency (ns)": 52310055, "99.90 percentile latency (ns)": 67359457, "Max latency (ns)": 76203254, "Mean latency (ns)": 25056811, "Min duration satisfied": "Yes", "Min latency (ns)": 4174982, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 39.82, "QPS w/o loadgen overhead": 39.91, "Result is": "VALID", "SUT name": "GNMT SERVER", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA Tesla T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.90th_percentile_latency_ms": 41.322309, "characteristics.90th_percentile_latency_ns": 41322309.0, "characteristics.90th_percentile_latency_s": 0.041322309, "characteristics.90th_percentile_latency_us": 41322.309, "characteristics.blue": 23.8, "ck_system": "T4x8", "ck_used": false, "cooling": "", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "closed", "formal_model": "nmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 6.0, CUDA 10.1, cuDNN 7.6.3, libjemalloc2, cub 1.8.0, tensorrt-laboratory mlperf branch", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.5, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/results/T4x8", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.3", "other_software_stack": "docker 18.09.2, python 3.6.8,gcc 5.5.0,onnx 1.3.0, tensorflow 1.13.1, pytorch 1.1.0, torchvision 0.3.0, pycuda 2019.1, sacrebleu 1.3.3, SimpleJSON, OpenCV 4.1.1", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 1, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x8", "system_name": "Supermicro 4029GP-TRT-OTO-28 8xT4", "target_latency (ns)": 0, "target_qps": 20.3603, "task": "NMT", "task2": "nmt", "total_cores": 56, "uid": "4a0bf4d3789df351", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" } ]