[ { "50.00 percentile latency (ns)": 170335804546, "90.00 percentile latency (ns)": 246046710459, "95.00 percentile latency (ns)": 252134038006, "97.00 percentile latency (ns)": 254560690639, "99.00 percentile latency (ns)": 256541254281, "99.90 percentile latency (ns)": 256966693855, "Max latency (ns)": 256966693855, "Mean latency (ns)": 157079270027, "Min duration satisfied": "Yes", "Min latency (ns)": 3575983154, "Min queries satisfied": "Yes", "Mode": "Performance", "Result is": "VALID", "SUT name": "GNMT SERVER", "Samples per second": 95.6389, "Scenario": "offline", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier", "accelerator_on-chip_memories": "1MB (128KB/SM) L1 + 512KB L2 + 4MB (DLA)", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.blue": 23.9, "characteristics.samples_per_second": 95.6389, "characteristics.samples_per_second.normalized_per_core": 95.6389, "characteristics.samples_per_second.normalized_per_processor": 95.6389, "ck_system": "Xavier", "ck_used": true, "cooling": "", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "open", "formal_model": "gnmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.3 DP, TensorRT 6.0, cuDNN 7.6.3, CUDA 10.0, cub 1.8.0", "host_memory_capacity": "16 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "8MB L2 (2MB per dual cluster)/4 MB L3 (shared)", "host_processor_core_count": 120, "host_processor_frequency": "2265.5 MHz", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1, "mlperf_version": 0.5, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/open/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/open/NVIDIA/results/Xavier", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "pycuda 2019.1, pytorch 1.1, torchvision 0.2.2.post3; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 24576, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "target_latency (ns)": 0, "target_qps": 100, "task": "NMT", "task2": "nmt", "total_cores": 120, "uid": "ae88e247dcb85bfc", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 42970414629, "90.00 percentile latency (ns)": 61995114746, "95.00 percentile latency (ns)": 63399549051, "97.00 percentile latency (ns)": 63949836381, "99.00 percentile latency (ns)": 64462529860, "99.90 percentile latency (ns)": 64605092935, "Max latency (ns)": 64625796888, "Mean latency (ns)": 39678455419, "Min duration satisfied": "Yes", "Min latency (ns)": 831171392, "Min queries satisfied": "Yes", "Mode": "Performance", "Result is": "VALID", "SUT name": "GNMT SERVER", "Samples per second": 4242.33, "Scenario": "offline", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA TITAN RTX", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.blue": 23.7, "characteristics.samples_per_second": 4242.33, "characteristics.samples_per_second.normalized_per_core": 1060.5825, "characteristics.samples_per_second.normalized_per_processor": 1060.5825, "ck_system": "TitanRTXx4", "ck_used": false, "cooling": "watercooled", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "closed", "formal_model": "nmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 6.0, CUDA 10.1, cuDNN 7.6.3, libjemalloc2, cub 1.8.0, tensorrt-laboratory mlperf branch", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 24, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) 8268", "host_processors_per_node": 2, "host_storage_capacity": "3.84 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1, "mlperf_version": 0.5, "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/results/TitanRTXx4", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.3", "other_software_stack": "docker 18.09.2, python 3.6.8,gcc 5.5.0,onnx 1.3.0, tensorflow 1.13.1, pytorch 1.1.0, torchvision 0.3.0, pycuda 2019.1, sacrebleu 1.3.3, SimpleJSON, OpenCV 4.1.1", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 274164, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/TitanRTXx4", "system_name": "SCAN 3XS DBP T496X2 Fluid", "target_latency (ns)": 0, "target_qps": 4154, "task": "NMT", "task2": "nmt", "total_cores": 48, "uid": "94432e8cb68bbb86", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41891694056, "90.00 percentile latency (ns)": 60704647984, "95.00 percentile latency (ns)": 62019643790, "97.00 percentile latency (ns)": 62557712262, "99.00 percentile latency (ns)": 63061304483, "99.90 percentile latency (ns)": 63181413719, "Max latency (ns)": 63187617187, "Mean latency (ns)": 38813154098, "Min duration satisfied": "Yes", "Min latency (ns)": 1651309269, "Min queries satisfied": "Yes", "Mode": "Performance", "Result is": "VALID", "SUT name": "GNMT SERVER", "Samples per second": 7154.88, "Scenario": "offline", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA Tesla T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 20, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.blue": 23.8, "characteristics.samples_per_second": 7154.88, "characteristics.samples_per_second.normalized_per_core": 357.744, "characteristics.samples_per_second.normalized_per_processor": 357.744, "ck_system": "T4x20", "ck_used": false, "cooling": "", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "closed", "formal_model": "nmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 6.0, CUDA 10.1, cuDNN 7.6.3, libjemalloc2, cub 1.8.0, tensorrt-laboratory mlperf branch", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1, "mlperf_version": 0.5, "normalize_cores": 20, "normalize_processors": 20, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/results/T4x20", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.3", "other_software_stack": "docker 18.09.2, python 3.6.8,gcc 5.5.0,onnx 1.3.0, tensorflow 1.13.1, pytorch 1.1.0, torchvision 0.3.0, pycuda 2019.1, sacrebleu 1.3.3, SimpleJSON, OpenCV 4.1.1", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 452100, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x20", "system_name": "Supermicro 6049GP-TRT-OTO-29 20xT4", "target_latency (ns)": 0, "target_qps": 6850, "task": "NMT", "task2": "nmt", "total_cores": 56, "uid": "27ab535c6f4bc9ab", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41775917556, "90.00 percentile latency (ns)": 60788063072, "95.00 percentile latency (ns)": 62083282955, "97.00 percentile latency (ns)": 62710079679, "99.00 percentile latency (ns)": 63178466684, "99.90 percentile latency (ns)": 63305133967, "Max latency (ns)": 63305133967, "Mean latency (ns)": 38769581017, "Min duration satisfied": "Yes", "Min latency (ns)": 1547790008, "Min queries satisfied": "Yes", "Mode": "Performance", "Result is": "VALID", "SUT name": "GNMT SERVER", "Samples per second": 2834.75, "Scenario": "offline", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA Tesla T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "characteristics.blue": 23.8, "characteristics.samples_per_second": 2834.75, "characteristics.samples_per_second.normalized_per_core": 354.34375, "characteristics.samples_per_second.normalized_per_processor": 354.34375, "ck_system": "T4x8", "ck_used": false, "cooling": "", "dataset": "WMT E-G", "dataset_link": "", "dim_x_default": "characteristics.samples_per_second", "dim_x_maximize": true, "dim_y_default": "characteristics.blue", "dim_y_maximize": true, "division": "closed", "formal_model": "nmt", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 6.0, CUDA 10.1, cuDNN 7.6.3, libjemalloc2, cub 1.8.0, tensorrt-laboratory mlperf branch", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "gnmt", "input_data_types": "UTF-8", "key.accuracy": "characteristics.blue", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1, "mlperf_version": 0.5, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.5/tree/master/closed/NVIDIA/results/T4x8", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.3", "other_software_stack": "docker 18.09.2, python 3.6.8,gcc 5.5.0,onnx 1.3.0, tensorflow 1.13.1, pytorch 1.1.0, torchvision 0.3.0, pycuda 2019.1, sacrebleu 1.3.3, SimpleJSON, OpenCV 4.1.1", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 3903900, "print_timestamps": true, "problem": false, "qsl_rng_seed": 3133965575612453542, "retraining": "N", "sample_index_rng_seed": 665484352860916858, "samples_per_query": 179454, "schedule_rng_seed": 3622009729038561421, "starting_weights_filename": "ende_gnmt_model_4_layer/translate.ckpt.data-00000-of-00001", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x8", "system_name": "Supermicro 4029GP-TRT-OTO-28 8xT4", "target_latency (ns)": 0, "target_qps": 2719, "task": "NMT", "task2": "nmt", "total_cores": 56, "uid": "b562f34e94895ae9", "use_accelerator": true, "weight_data_types": "int8,fp16", "weight_transformations": "quantization, affine fusion" } ]