[ { "50.00 percentile latency (ns)": 20714556, "90.00 percentile latency (ns)": 27437756, "95.00 percentile latency (ns)": 30441904, "97.00 percentile latency (ns)": 32732446, "99.00 percentile latency (ns)": 38459627, "99.90 percentile latency (ns)": 47816192, "Completed samples per second": 7001.88, "Max latency (ns)": 59641164, "Mean latency (ns)": 21842489, "Min duration satisfied": "Yes", "Min latency (ns)": 11974143, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 7002.11, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 7002.11, "characteristics.scheduled_queries_per_second.normalized_per_core": 875.26375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 875.26375, "ck_system": "A10x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7000, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "6762fed4933a25aa", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 20170151, "90.00 percentile latency (ns)": 24619751, "95.00 percentile latency (ns)": 26452056, "97.00 percentile latency (ns)": 27817969, "99.00 percentile latency (ns)": 30811807, "99.90 percentile latency (ns)": 36715387, "Completed samples per second": 3196.94, "Max latency (ns)": 1007875118, "Mean latency (ns)": 20566281, "Min duration satisfied": "Yes", "Min latency (ns)": 10716597, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 3202.31, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 3202.31, "characteristics.scheduled_queries_per_second.normalized_per_core": 400.28875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 400.28875, "ck_system": "A10x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3200, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "074be1833f9672a0", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 59321860, "90.00 percentile latency (ns)": 95952045, "95.00 percentile latency (ns)": 99645597, "97.00 percentile latency (ns)": 104607300, "99.00 percentile latency (ns)": 116430313, "99.90 percentile latency (ns)": 149232451, "Completed samples per second": 20462.4, "Max latency (ns)": 1029754868, "Mean latency (ns)": 64253837, "Min duration satisfied": "Yes", "Min latency (ns)": 17818118, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_MultiMigServer", "Scenario": "server", "Scheduled samples per second": 20496.68, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (7x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 20496.68, "characteristics.scheduled_queries_per_second.normalized_per_core": 2562.085, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2562.085, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB-MIG-7x1g.10gb, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 20500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "6d32f4ff9aaf8de1", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 75955660, "90.00 percentile latency (ns)": 114193575, "95.00 percentile latency (ns)": 118698252, "97.00 percentile latency (ns)": 122118860, "99.00 percentile latency (ns)": 129356008, "99.90 percentile latency (ns)": 148278976, "Completed samples per second": 9285.52, "Max latency (ns)": 1041316141, "Mean latency (ns)": 76352324, "Min duration satisfied": "Yes", "Min latency (ns)": 20893977, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_MultiMigServer", "Scenario": "server", "Scheduled samples per second": 9301.26, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (7x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 9301.26, "characteristics.scheduled_queries_per_second.normalized_per_core": 1162.6575, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1162.6575, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB-MIG-7x1g.10gb, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 9300, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "00a67cdbcd29ebba", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 44249153, "90.00 percentile latency (ns)": 51837905, "95.00 percentile latency (ns)": 54737682, "97.00 percentile latency (ns)": 57059544, "99.00 percentile latency (ns)": 62379293, "99.90 percentile latency (ns)": 74648283, "Completed samples per second": 10783.64, "Max latency (ns)": 1020178957, "Mean latency (ns)": 44477515, "Min duration satisfied": "Yes", "Min latency (ns)": 21086085, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 10801.9, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 10801.9, "characteristics.scheduled_queries_per_second.normalized_per_core": 2700.475, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2700.475, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT_Triton", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10800, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "c7c5431ca8b47ace", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 52889850, "90.00 percentile latency (ns)": 59119437, "95.00 percentile latency (ns)": 60649401, "97.00 percentile latency (ns)": 61648681, "99.00 percentile latency (ns)": 63524973, "99.90 percentile latency (ns)": 66742716, "Completed samples per second": 4643.6, "Max latency (ns)": 1019348410, "Mean latency (ns)": 52986053, "Min duration satisfied": "Yes", "Min latency (ns)": 37887137, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 4651.45, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 4651.45, "characteristics.scheduled_queries_per_second.normalized_per_core": 1162.8625, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1162.8625, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT_Triton", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4650, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "847424889972e04e", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 47978178, "90.00 percentile latency (ns)": 82382857, "95.00 percentile latency (ns)": 84700656, "97.00 percentile latency (ns)": 86084667, "99.00 percentile latency (ns)": 88793824, "99.90 percentile latency (ns)": 105892428, "Completed samples per second": 8301.01, "Max latency (ns)": 1019838328, "Mean latency (ns)": 50770136, "Min duration satisfied": "Yes", "Min latency (ns)": 14753461, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_MultiMigServer", "Scenario": "server", "Scheduled samples per second": 8301.45, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (4x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 8301.45, "characteristics.scheduled_queries_per_second.normalized_per_core": 1037.68125, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1037.68125, "ck_system": "A30-MIG_32x1g.6gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_32x1g.6gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_32x1g.6gb_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A30-MIG-4x1g.6gb, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 8300, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "54d919bc9ef4f874", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 72401024, "90.00 percentile latency (ns)": 105961589, "95.00 percentile latency (ns)": 111112133, "97.00 percentile latency (ns)": 115330782, "99.00 percentile latency (ns)": 128486163, "99.90 percentile latency (ns)": 183689948, "Completed samples per second": 3796.45, "Max latency (ns)": 1033017291, "Mean latency (ns)": 72242881, "Min duration satisfied": "Yes", "Min latency (ns)": 17416106, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_MultiMigServer", "Scenario": "server", "Scheduled samples per second": 3802.86, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (4x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 3802.86, "characteristics.scheduled_queries_per_second.normalized_per_core": 475.3575, "characteristics.scheduled_queries_per_second.normalized_per_processor": 475.3575, "ck_system": "A30-MIG_32x1g.6gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_32x1g.6gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_32x1g.6gb_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A30-MIG-4x1g.6gb, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3800, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "0bc7e8a8fc1443c9", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 33694640, "90.00 percentile latency (ns)": 47262543, "95.00 percentile latency (ns)": 51307179, "97.00 percentile latency (ns)": 53929598, "99.00 percentile latency (ns)": 58903546, "99.90 percentile latency (ns)": 68060577, "Completed samples per second": 10401.74, "Max latency (ns)": 82453072, "Mean latency (ns)": 34312713, "Min duration satisfied": "Yes", "Min latency (ns)": 3019377, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10402.19, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 10402.19, "characteristics.scheduled_queries_per_second.normalized_per_core": 2600.5475, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2600.5475, "ck_system": "A100-PCIe-80GB_aarch64x4_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.2, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 80, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Ampere Altra Q80-30", "host_processors_per_node": 1, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GB_aarch64x4_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.2, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GB_aarch64x4_TRT", "system_name": "Gigabyte G242-P31 (4x A100-PCIe-80GB_aarch64, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10400, "task": "NLP", "task2": "nlp", "total_cores": 80, "uid": "71430b82e4f8de5b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 23221319, "90.00 percentile latency (ns)": 33523552, "95.00 percentile latency (ns)": 36994592, "97.00 percentile latency (ns)": 39407171, "99.00 percentile latency (ns)": 44168893, "99.90 percentile latency (ns)": 54592457, "Completed samples per second": 4801.79, "Max latency (ns)": 71805992, "Mean latency (ns)": 23979524, "Min duration satisfied": "Yes", "Min latency (ns)": 3877987, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 4801.89, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 4801.89, "characteristics.scheduled_queries_per_second.normalized_per_core": 1200.4725, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1200.4725, "ck_system": "A100-PCIe-80GB_aarch64x4_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.2, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 80, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Ampere Altra Q80-30", "host_processors_per_node": 1, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GB_aarch64x4_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.2, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GB_aarch64x4_TRT", "system_name": "Gigabyte G242-P31 (4x A100-PCIe-80GB_aarch64, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4800, "task": "NLP", "task2": "nlp", "total_cores": 80, "uid": "888f17d24c55b705", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 47499325, "90.00 percentile latency (ns)": 70648744, "95.00 percentile latency (ns)": 78220856, "97.00 percentile latency (ns)": 83512467, "99.00 percentile latency (ns)": 97623998, "99.90 percentile latency (ns)": 512074126232, "Completed samples per second": 11500.92, "Max latency (ns)": 604054788366, "Mean latency (ns)": 2118083194, "Min duration satisfied": "Yes", "Min latency (ns)": 5261791, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 11501.43, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 11501.43, "characteristics.scheduled_queries_per_second.normalized_per_core": 1437.67875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1437.67875, "ck_system": "A30x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "fd2ac4cf089c3a6b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 46651240, "90.00 percentile latency (ns)": 74200662, "95.00 percentile latency (ns)": 83546536, "97.00 percentile latency (ns)": 90242007, "99.00 percentile latency (ns)": 109083927, "99.90 percentile latency (ns)": 509221267206, "Completed samples per second": 5251.97, "Max latency (ns)": 603474370445, "Mean latency (ns)": 2146737530, "Min duration satisfied": "Yes", "Min latency (ns)": 5685825, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 5252.86, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 5252.86, "characteristics.scheduled_queries_per_second.normalized_per_core": 656.6075, "characteristics.scheduled_queries_per_second.normalized_per_processor": 656.6075, "ck_system": "A30x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 5250, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "06d3b21e427cbc95", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26466562, "90.00 percentile latency (ns)": 45210850, "95.00 percentile latency (ns)": 47154216, "97.00 percentile latency (ns)": 48226926, "99.00 percentile latency (ns)": 50129963, "99.90 percentile latency (ns)": 53957009, "Completed samples per second": 11001.45, "Max latency (ns)": 91584163, "Mean latency (ns)": 27560767, "Min duration satisfied": "Yes", "Min latency (ns)": 5031659, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 11001.55, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 11001.55, "characteristics.scheduled_queries_per_second.normalized_per_core": 1375.19375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1375.19375, "ck_system": "A30x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "18367d4c2c8d808b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 91549139, "90.00 percentile latency (ns)": 98725386, "95.00 percentile latency (ns)": 100686272, "97.00 percentile latency (ns)": 101980416, "99.00 percentile latency (ns)": 104560693, "99.90 percentile latency (ns)": 109367242, "Completed samples per second": 5193.87, "Max latency (ns)": 1054661792, "Mean latency (ns)": 91601026, "Min duration satisfied": "Yes", "Min latency (ns)": 70016875, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 5202.95, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 5202.95, "characteristics.scheduled_queries_per_second.normalized_per_core": 650.36875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 650.36875, "ck_system": "A30x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 5200, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "19bb381b458fb6ab", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 55338304, "90.00 percentile latency (ns)": 72280085, "95.00 percentile latency (ns)": 76638619, "97.00 percentile latency (ns)": 79512646, "99.00 percentile latency (ns)": 84213340, "99.90 percentile latency (ns)": 90326373, "Completed samples per second": 22992.17, "Max latency (ns)": 116823943, "Mean latency (ns)": 55138793, "Min duration satisfied": "Yes", "Min latency (ns)": 3382956, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 22993.76, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 22993.76, "characteristics.scheduled_queries_per_second.normalized_per_core": 2874.22, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2874.22, "ck_system": "A100-PCIe-80GBx8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx8_TRT", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 23000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "09a4463e99b581e9", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 35355065, "90.00 percentile latency (ns)": 55373428, "95.00 percentile latency (ns)": 62047890, "97.00 percentile latency (ns)": 66643912, "99.00 percentile latency (ns)": 75122706, "99.90 percentile latency (ns)": 90503155, "Completed samples per second": 10801.38, "Max latency (ns)": 120948969, "Mean latency (ns)": 37198703, "Min duration satisfied": "Yes", "Min latency (ns)": 4113454, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10801.9, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 10801.9, "characteristics.scheduled_queries_per_second.normalized_per_core": 1350.2375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1350.2375, "ck_system": "A100-PCIe-80GBx8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx8_TRT", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10800, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "1b5d9190766037ac", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 49067439, "90.00 percentile latency (ns)": 70366825, "95.00 percentile latency (ns)": 76731613, "97.00 percentile latency (ns)": 80992725, "99.00 percentile latency (ns)": 88995055, "99.90 percentile latency (ns)": 101318894, "Completed samples per second": 17495.46, "Max latency (ns)": 123026766, "Mean latency (ns)": 50080723, "Min duration satisfied": "Yes", "Min latency (ns)": 5116172, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 17496.25, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 2188.9636666666665, "characteristics.power.normalized_per_core": 273.6204583333333, "characteristics.power.normalized_per_processor": 273.6204583333333, "characteristics.scheduled_queries_per_second": 17496.25, "characteristics.scheduled_queries_per_second.normalized_per_core": 2187.03125, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2187.03125, "ck_system": "A100-PCIex8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT_MaxQ", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 17500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "b59a89e3e2ab7c0a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 24771738, "90.00 percentile latency (ns)": 37248701, "95.00 percentile latency (ns)": 41742782, "97.00 percentile latency (ns)": 44923959, "99.00 percentile latency (ns)": 51613718, "99.90 percentile latency (ns)": 65327713, "Completed samples per second": 7500.87, "Max latency (ns)": 102474803, "Mean latency (ns)": 26043744, "Min duration satisfied": "Yes", "Min latency (ns)": 5353216, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 7501.09, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 2148.7096666666685, "characteristics.power.normalized_per_core": 268.58870833333356, "characteristics.power.normalized_per_processor": 268.58870833333356, "characteristics.scheduled_queries_per_second": 7501.09, "characteristics.scheduled_queries_per_second.normalized_per_core": 937.63625, "characteristics.scheduled_queries_per_second.normalized_per_processor": 937.63625, "ck_system": "A100-PCIex8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT_MaxQ", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "6895406eb602c007", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 57779817, "90.00 percentile latency (ns)": 64526851, "95.00 percentile latency (ns)": 66329692, "97.00 percentile latency (ns)": 67678140, "99.00 percentile latency (ns)": 78174890, "99.90 percentile latency (ns)": 545337912588, "Completed samples per second": 25792.95, "Max latency (ns)": 606769867031, "Mean latency (ns)": 3065887580, "Min duration satisfied": "Yes", "Min latency (ns)": 5401323, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 25795.37, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 25795.37, "characteristics.scheduled_queries_per_second.normalized_per_core": 3224.42125, "characteristics.scheduled_queries_per_second.normalized_per_processor": 3224.42125, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 25800, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "d1b7286665143cb5", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 59802850, "90.00 percentile latency (ns)": 68043741, "95.00 percentile latency (ns)": 71218436, "97.00 percentile latency (ns)": 74037919, "99.00 percentile latency (ns)": 88085628, "99.90 percentile latency (ns)": 545962996109, "Completed samples per second": 13101.02, "Max latency (ns)": 607128162328, "Mean latency (ns)": 3072298620, "Min duration satisfied": "Yes", "Min latency (ns)": 4840547, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 13101.7, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 13101.7, "characteristics.scheduled_queries_per_second.normalized_per_core": 1637.7125, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1637.7125, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 13100, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "c938c247e55ee750", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 76277329, "90.00 percentile latency (ns)": 98447018, "95.00 percentile latency (ns)": 103331088, "97.00 percentile latency (ns)": 106300549, "99.00 percentile latency (ns)": 111632640, "99.90 percentile latency (ns)": 119222154, "Completed samples per second": 10800.82, "Max latency (ns)": 131612451, "Mean latency (ns)": 75778283, "Min duration satisfied": "Yes", "Min latency (ns)": 6708242, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10801.9, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 10801.9, "characteristics.scheduled_queries_per_second.normalized_per_core": 2700.475, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2700.475, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10800, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "4ca89a9c7ccde0c2", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 37538629, "90.00 percentile latency (ns)": 59610959, "95.00 percentile latency (ns)": 67621829, "97.00 percentile latency (ns)": 73218087, "99.00 percentile latency (ns)": 84717714, "99.90 percentile latency (ns)": 107535754, "Completed samples per second": 4861.51, "Max latency (ns)": 135167708, "Mean latency (ns)": 39810126, "Min duration satisfied": "Yes", "Min latency (ns)": 4151477, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 4861.86, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 4861.86, "characteristics.scheduled_queries_per_second.normalized_per_core": 1215.465, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1215.465, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4860, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "3c340a494c3f83f7", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 37265023, "90.00 percentile latency (ns)": 56911260, "95.00 percentile latency (ns)": 63803025, "97.00 percentile latency (ns)": 69433838, "99.00 percentile latency (ns)": 108925390, "99.90 percentile latency (ns)": 721311782411, "Completed samples per second": 339.92, "Max latency (ns)": 800298850187, "Mean latency (ns)": 3976899723, "Min duration satisfied": "Yes", "Min latency (ns)": 4273084, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 339.93, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (1x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 339.93, "characteristics.scheduled_queries_per_second.normalized_per_core": 339.93, "characteristics.scheduled_queries_per_second.normalized_per_processor": 339.93, "ck_system": "A30-MIG_1x1g.6gb_TRT_HeteroMultiUse", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_1x1g.6gb_TRT_HeteroMultiUse", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_1x1g.6gb_TRT_HeteroMultiUse", "system_name": "Gigabyte G482-Z54 (1x A30-MIG-1x1g.6gb, TensorRT, HeteroMultiUse)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 340, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "04341b1c64b70a14", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 36437968, "90.00 percentile latency (ns)": 61081083, "95.00 percentile latency (ns)": 69320871, "97.00 percentile latency (ns)": 75660288, "99.00 percentile latency (ns)": 123244235, "99.90 percentile latency (ns)": 1873122402990, "Completed samples per second": 129.97, "Max latency (ns)": 2091957356595, "Mean latency (ns)": 10391088579, "Min duration satisfied": "Yes", "Min latency (ns)": 6533515, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 129.97, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (1x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 129.97, "characteristics.scheduled_queries_per_second.normalized_per_core": 129.97, "characteristics.scheduled_queries_per_second.normalized_per_processor": 129.97, "ck_system": "A30-MIG_1x1g.6gb_TRT_HeteroMultiUse", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_1x1g.6gb_TRT_HeteroMultiUse", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_1x1g.6gb_TRT_HeteroMultiUse", "system_name": "Gigabyte G482-Z54 (1x A30-MIG-1x1g.6gb, TensorRT, HeteroMultiUse)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 130, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "365e9542fd9aeec9", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41582953, "90.00 percentile latency (ns)": 63069902, "95.00 percentile latency (ns)": 69667186, "97.00 percentile latency (ns)": 74457099, "99.00 percentile latency (ns)": 91880236, "99.90 percentile latency (ns)": 625720679164, "Completed samples per second": 379.91, "Max latency (ns)": 714823728784, "Mean latency (ns)": 3366250407, "Min duration satisfied": "Yes", "Min latency (ns)": 4397814, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 379.93, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 379.93, "characteristics.scheduled_queries_per_second.normalized_per_core": 379.93, "characteristics.scheduled_queries_per_second.normalized_per_processor": 379.93, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, HeteroMultiUse)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 380, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "46b8b121f89217a2", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43953984, "90.00 percentile latency (ns)": 69204352, "95.00 percentile latency (ns)": 77566711, "97.00 percentile latency (ns)": 83599809, "99.00 percentile latency (ns)": 112370439, "99.90 percentile latency (ns)": 1496546258103, "Completed samples per second": 159.97, "Max latency (ns)": 1696631381204, "Mean latency (ns)": 8178403832, "Min duration satisfied": "Yes", "Min latency (ns)": 6721329, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 159.97, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 159.97, "characteristics.scheduled_queries_per_second.normalized_per_core": 159.97, "characteristics.scheduled_queries_per_second.normalized_per_processor": 159.97, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, HeteroMultiUse)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 160, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "68ebd8674a8433d9", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 67409172, "90.00 percentile latency (ns)": 90935134, "95.00 percentile latency (ns)": 97632708, "97.00 percentile latency (ns)": 101853937, "99.00 percentile latency (ns)": 108970200, "99.90 percentile latency (ns)": 119912324, "Completed samples per second": 10201.65, "Max latency (ns)": 135899913, "Mean latency (ns)": 67715047, "Min duration satisfied": "Yes", "Min latency (ns)": 6503530, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10202.62, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 1274.3638333333322, "characteristics.power.normalized_per_core": 318.59095833333305, "characteristics.power.normalized_per_processor": 318.59095833333305, "characteristics.scheduled_queries_per_second": 10202.62, "characteristics.scheduled_queries_per_second.normalized_per_core": 2550.655, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2550.655, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10200, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "e0dde17747e5b8ed", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 24410626, "90.00 percentile latency (ns)": 35756825, "95.00 percentile latency (ns)": 39704903, "97.00 percentile latency (ns)": 42493288, "99.00 percentile latency (ns)": 48220693, "99.90 percentile latency (ns)": 59792585, "Completed samples per second": 4301.57, "Max latency (ns)": 89992094, "Mean latency (ns)": 25374318, "Min duration satisfied": "Yes", "Min latency (ns)": 4294289, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 4301.74, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 1263.3221297836935, "characteristics.power.normalized_per_core": 315.8305324459234, "characteristics.power.normalized_per_processor": 315.8305324459234, "characteristics.scheduled_queries_per_second": 4301.74, "characteristics.scheduled_queries_per_second.normalized_per_core": 1075.435, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1075.435, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4300, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "ae846b389df81400", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 60303636, "90.00 percentile latency (ns)": 79212304, "95.00 percentile latency (ns)": 84142684, "97.00 percentile latency (ns)": 87273714, "99.00 percentile latency (ns)": 92635739, "99.90 percentile latency (ns)": 99343939, "Completed samples per second": 20795.42, "Max latency (ns)": 127749854, "Mean latency (ns)": 60114155, "Min duration satisfied": "Yes", "Min latency (ns)": 5674114, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 20796.88, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 20796.88, "characteristics.scheduled_queries_per_second.normalized_per_core": 2599.61, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2599.61, "ck_system": "A100-PCIex8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 20800, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "6d83e5b3b6044bc4", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 38419345, "90.00 percentile latency (ns)": 60013764, "95.00 percentile latency (ns)": 67332669, "97.00 percentile latency (ns)": 72123447, "99.00 percentile latency (ns)": 81589672, "99.90 percentile latency (ns)": 98504666, "Completed samples per second": 9601.29, "Max latency (ns)": 150310626, "Mean latency (ns)": 40388288, "Min duration satisfied": "Yes", "Min latency (ns)": 5476197, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 9601.56, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 9601.56, "characteristics.scheduled_queries_per_second.normalized_per_core": 1200.195, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1200.195, "ck_system": "A100-PCIex8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 9600, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "b305aee2d4ed3b0f", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26544512, "90.00 percentile latency (ns)": 28661020, "95.00 percentile latency (ns)": 29273232, "97.00 percentile latency (ns)": 29682874, "99.00 percentile latency (ns)": 30510313, "99.90 percentile latency (ns)": 36599583, "Completed samples per second": 17966.4, "Max latency (ns)": 1014874702, "Mean latency (ns)": 26609074, "Min duration satisfied": "Yes", "Min latency (ns)": 18402691, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 17996.75, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 17996.75, "characteristics.scheduled_queries_per_second.normalized_per_core": 2249.59375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2249.59375, "ck_system": "A100-PCIex8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 18000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "a5ae7489caa00d0d", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 52120757, "90.00 percentile latency (ns)": 56441813, "95.00 percentile latency (ns)": 57846477, "97.00 percentile latency (ns)": 58775392, "99.00 percentile latency (ns)": 60467507, "99.90 percentile latency (ns)": 65294150, "Completed samples per second": 9485.96, "Max latency (ns)": 1016738046, "Mean latency (ns)": 52271819, "Min duration satisfied": "Yes", "Min latency (ns)": 38483935, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 9502.0, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 9502.0, "characteristics.scheduled_queries_per_second.normalized_per_core": 1187.75, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1187.75, "ck_system": "A100-PCIex8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 9500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "4c1592f5cfff18fc", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 19101093, "90.00 percentile latency (ns)": 22902808, "95.00 percentile latency (ns)": 24429874, "97.00 percentile latency (ns)": 25558772, "99.00 percentile latency (ns)": 27992843, "99.90 percentile latency (ns)": 33032479, "Completed samples per second": 22993.11, "Max latency (ns)": 59353423, "Mean latency (ns)": 19600910, "Min duration satisfied": "Yes", "Min latency (ns)": 12713635, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 22993.76, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 22993.76, "characteristics.scheduled_queries_per_second.normalized_per_core": 2874.22, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2874.22, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 23000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "a1e068c6f2645c4a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 42811785, "90.00 percentile latency (ns)": 46097985, "95.00 percentile latency (ns)": 47213674, "97.00 percentile latency (ns)": 47950246, "99.00 percentile latency (ns)": 49109063, "99.90 percentile latency (ns)": 50840858, "Completed samples per second": 11186.9, "Max latency (ns)": 1032406502, "Mean latency (ns)": 42910454, "Min duration satisfied": "Yes", "Min latency (ns)": 33268293, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 11206.06, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 11206.06, "characteristics.scheduled_queries_per_second.normalized_per_core": 1400.7575, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1400.7575, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11205, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "840259faed4d2fbe", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 53073709, "90.00 percentile latency (ns)": 65964005, "95.00 percentile latency (ns)": 69116960, "97.00 percentile latency (ns)": 70905196, "99.00 percentile latency (ns)": 79074374, "99.90 percentile latency (ns)": 545938199569, "Completed samples per second": 21494.93, "Max latency (ns)": 606534158710, "Mean latency (ns)": 3056261004, "Min duration satisfied": "Yes", "Min latency (ns)": 4799910, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 21496.78, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 3454.9475247524747, "characteristics.power.normalized_per_core": 431.86844059405934, "characteristics.power.normalized_per_processor": 431.86844059405934, "characteristics.scheduled_queries_per_second": 21496.78, "characteristics.scheduled_queries_per_second.normalized_per_core": 2687.0975, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2687.0975, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 21500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "b4d374e578a146c0", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 34248684, "90.00 percentile latency (ns)": 47619105, "95.00 percentile latency (ns)": 51920726, "97.00 percentile latency (ns)": 55164529, "99.00 percentile latency (ns)": 71069625, "99.90 percentile latency (ns)": 545597312664, "Completed samples per second": 10001.49, "Max latency (ns)": 606637656211, "Mean latency (ns)": 3045892883, "Min duration satisfied": "Yes", "Min latency (ns)": 4739467, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10001.94, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 3409.112520593078, "characteristics.power.normalized_per_core": 426.13906507413475, "characteristics.power.normalized_per_processor": 426.13906507413475, "characteristics.scheduled_queries_per_second": 10001.94, "characteristics.scheduled_queries_per_second.normalized_per_core": 1250.2425, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1250.2425, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "system_name": "NVIDIA DGX A100 (8x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "124cb471d955a218", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 24043950, "90.00 percentile latency (ns)": 25999029, "95.00 percentile latency (ns)": 26562159, "97.00 percentile latency (ns)": 26930816, "99.00 percentile latency (ns)": 27642527, "99.90 percentile latency (ns)": 29359840, "Completed samples per second": 17966.43, "Max latency (ns)": 1013844718, "Mean latency (ns)": 24089540, "Min duration satisfied": "Yes", "Min latency (ns)": 16401277, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 17996.75, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 17996.75, "characteristics.scheduled_queries_per_second.normalized_per_core": 2249.59375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2249.59375, "ck_system": "A100-PCIe-80GBx8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 18000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "6d72ed51f66b495f", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 45874293, "90.00 percentile latency (ns)": 49679290, "95.00 percentile latency (ns)": 50870472, "97.00 percentile latency (ns)": 51661787, "99.00 percentile latency (ns)": 53037542, "99.90 percentile latency (ns)": 55122776, "Completed samples per second": 9485.99, "Max latency (ns)": 1014849000, "Mean latency (ns)": 45963079, "Min duration satisfied": "Yes", "Min latency (ns)": 34579457, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 9502.0, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 9502.0, "characteristics.scheduled_queries_per_second.normalized_per_core": 1187.75, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1187.75, "ck_system": "A100-PCIe-80GBx8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 9500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "c4455da0f9fa07a0", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43270571, "90.00 percentile latency (ns)": 56493560, "95.00 percentile latency (ns)": 60194969, "97.00 percentile latency (ns)": 62827661, "99.00 percentile latency (ns)": 71446983, "99.90 percentile latency (ns)": 519863633759, "Completed samples per second": 7201.21, "Max latency (ns)": 604945769690, "Mean latency (ns)": 2205067314, "Min duration satisfied": "Yes", "Min latency (ns)": 4301315, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 7201.69, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 7201.69, "characteristics.scheduled_queries_per_second.normalized_per_core": 900.21125, "characteristics.scheduled_queries_per_second.normalized_per_processor": 900.21125, "ck_system": "A10x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7200, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "42d6a0a88a83b7f4", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26679870, "90.00 percentile latency (ns)": 38679036, "95.00 percentile latency (ns)": 42743329, "97.00 percentile latency (ns)": 45580325, "99.00 percentile latency (ns)": 53622234, "99.90 percentile latency (ns)": 519019865710, "Completed samples per second": 3122.4, "Max latency (ns)": 604282732715, "Mean latency (ns)": 2156614052, "Min duration satisfied": "Yes", "Min latency (ns)": 5511855, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 3122.63, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 3122.63, "characteristics.scheduled_queries_per_second.normalized_per_core": 390.32875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 390.32875, "ck_system": "A10x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3120, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "632228f824479bb4", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 73485830, "90.00 percentile latency (ns)": 87802516, "95.00 percentile latency (ns)": 103031540, "97.00 percentile latency (ns)": 111975099, "99.00 percentile latency (ns)": 128045919, "99.90 percentile latency (ns)": 163309260, "Completed samples per second": 26.49, "Max latency (ns)": 249865227, "Mean latency (ns)": 77029800, "Min duration satisfied": "Yes", "Min latency (ns)": 70813785, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 26.49, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "N/A", "accelerator_memory_configuration": "", "accelerator_model_name": "N/A", "accelerator_on-chip_memories": "", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 26.49, "characteristics.scheduled_queries_per_second.normalized_per_core": 0.23651785714285714, "characteristics.scheduled_queries_per_second.normalized_per_processor": 6.6225, "ck_system": "Triton_CPU_4S_8380Hx1", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "OpenVino 2021", "host_memory_capacity": "1536 GB", "host_memory_configuration": "12 slots / 32GB each / 3200 MT/s per socket", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Gold 8380H CPU @ 2.70GHz", "host_processors_per_node": 4, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "fp32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 112, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Triton_CPU_4S_8380Hx1", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "Tensorflow 2.4.0, OpenVino 2021, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "The original weight filename: bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "CPU Inference on Triton Inference Server", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Triton_CPU_4S_8380Hx1", "system_name": "Supermicro SYS-240P-TNRT (Cooper Lake running Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 26.5, "task": "NLP", "task2": "nlp", "total_cores": 112, "uid": "f4223beae32d17fb", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "We transfer the weight from int8 datatype in ONNX file to int8 datatype in OpenVino IR file." }, { "50.00 percentile latency (ns)": 40969807, "90.00 percentile latency (ns)": 77073635, "95.00 percentile latency (ns)": 86973333, "97.00 percentile latency (ns)": 94265153, "99.00 percentile latency (ns)": 116967189, "99.90 percentile latency (ns)": 128459020, "Completed samples per second": 7001.79, "Max latency (ns)": 136823451, "Mean latency (ns)": 45645281, "Min duration satisfied": "Yes", "Min latency (ns)": 11635535, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 7003.11, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 7003.11, "characteristics.scheduled_queries_per_second.normalized_per_core": 875.38875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 875.38875, "ck_system": "A10x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7000, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "a0da23a3b35bc9a8", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 21323557, "90.00 percentile latency (ns)": 27566486, "95.00 percentile latency (ns)": 30251218, "97.00 percentile latency (ns)": 32182761, "99.00 percentile latency (ns)": 36401507, "99.90 percentile latency (ns)": 45208188, "Completed samples per second": 3196.46, "Max latency (ns)": 1011571934, "Mean latency (ns)": 22078708, "Min duration satisfied": "Yes", "Min latency (ns)": 10923445, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 3201.83, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 3201.83, "characteristics.scheduled_queries_per_second.normalized_per_core": 400.22875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 400.22875, "ck_system": "A10x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3200, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "3b93bb0c64ab7804", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 59950491, "90.00 percentile latency (ns)": 96909506, "95.00 percentile latency (ns)": 101516031, "97.00 percentile latency (ns)": 108064254, "99.00 percentile latency (ns)": 119391355, "99.90 percentile latency (ns)": 154736654, "Completed samples per second": 20457.28, "Max latency (ns)": 1028543403, "Mean latency (ns)": 65253006, "Min duration satisfied": "Yes", "Min latency (ns)": 17379384, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_MultiMigServer", "Scenario": "server", "Scheduled samples per second": 20491.59, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (7x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 20491.59, "characteristics.scheduled_queries_per_second.normalized_per_core": 2561.44875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2561.44875, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB-MIG-7x1g.10gb, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 20500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "3d4dbdb21c209a86", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 75653292, "90.00 percentile latency (ns)": 113925221, "95.00 percentile latency (ns)": 118423591, "97.00 percentile latency (ns)": 121915729, "99.00 percentile latency (ns)": 129373528, "99.90 percentile latency (ns)": 151248618, "Completed samples per second": 9303.29, "Max latency (ns)": 1039359087, "Mean latency (ns)": 76219954, "Min duration satisfied": "Yes", "Min latency (ns)": 21270035, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_MultiMigServer", "Scenario": "server", "Scheduled samples per second": 9304.02, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (7x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 9304.02, "characteristics.scheduled_queries_per_second.normalized_per_core": 1163.0025, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1163.0025, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_56x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB-MIG-7x1g.10gb, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 9300, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "4a41791482aaadcf", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 62476028, "90.00 percentile latency (ns)": 82691100, "95.00 percentile latency (ns)": 90074400, "97.00 percentile latency (ns)": 95808801, "99.00 percentile latency (ns)": 120135445, "99.90 percentile latency (ns)": 544160214339, "Completed samples per second": 3201.46, "Max latency (ns)": 605928544101, "Mean latency (ns)": 3012272851, "Min duration satisfied": "Yes", "Min latency (ns)": 4756359, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 3201.83, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 3201.83, "characteristics.scheduled_queries_per_second.normalized_per_core": 3201.83, "characteristics.scheduled_queries_per_second.normalized_per_processor": 3201.83, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_datacenter", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_datacenter", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_datacenter", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3200, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "20906e17779ee224", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 38902247, "90.00 percentile latency (ns)": 53493169, "95.00 percentile latency (ns)": 58545594, "97.00 percentile latency (ns)": 62409861, "99.00 percentile latency (ns)": 81138185, "99.90 percentile latency (ns)": 538568022544, "Completed samples per second": 1552.35, "Max latency (ns)": 604926074593, "Mean latency (ns)": 2936096302, "Min duration satisfied": "Yes", "Min latency (ns)": 4472477, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 1552.5, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 1552.5, "characteristics.scheduled_queries_per_second.normalized_per_core": 1552.5, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1552.5, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_datacenter", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_datacenter", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_datacenter", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 1550, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "7a2b590c1469982f", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 27959067, "90.00 percentile latency (ns)": 36330547, "95.00 percentile latency (ns)": 39359755, "97.00 percentile latency (ns)": 41748913, "99.00 percentile latency (ns)": 46981694, "99.90 percentile latency (ns)": 57675581, "Completed samples per second": 2796.88, "Max latency (ns)": 1009868196, "Mean latency (ns)": 28209863, "Min duration satisfied": "Yes", "Min latency (ns)": 13080946, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 2801.56, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 2801.56, "characteristics.scheduled_queries_per_second.normalized_per_core": 2801.56, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2801.56, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_Triton_datacenter", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_Triton_datacenter", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_Triton_datacenter", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 2800, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "d2544983b4a3abd3", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 29032661, "90.00 percentile latency (ns)": 40717310, "95.00 percentile latency (ns)": 45882631, "97.00 percentile latency (ns)": 49651406, "99.00 percentile latency (ns)": 57947082, "99.90 percentile latency (ns)": 75910767, "Completed samples per second": 1400.67, "Max latency (ns)": 1006278550, "Mean latency (ns)": 29922975, "Min duration satisfied": "Yes", "Min latency (ns)": 11825470, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 1403.01, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 1403.01, "characteristics.scheduled_queries_per_second.normalized_per_core": 1403.01, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1403.01, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_Triton_datacenter", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_Triton_datacenter", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_Triton_datacenter", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 1400, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "1b3fbc95eea4718a", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 47162841, "90.00 percentile latency (ns)": 70362710, "95.00 percentile latency (ns)": 78009610, "97.00 percentile latency (ns)": 83408647, "99.00 percentile latency (ns)": 97836806, "99.90 percentile latency (ns)": 508010705245, "Completed samples per second": 11498.76, "Max latency (ns)": 603785539504, "Mean latency (ns)": 2134366448, "Min duration satisfied": "Yes", "Min latency (ns)": 4890960, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 11499.7, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 11499.7, "characteristics.scheduled_queries_per_second.normalized_per_core": 1437.4625, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1437.4625, "ck_system": "A30x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "669384acc973cf47", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 51959994, "90.00 percentile latency (ns)": 82903223, "95.00 percentile latency (ns)": 92865642, "97.00 percentile latency (ns)": 100346789, "99.00 percentile latency (ns)": 119970511, "99.90 percentile latency (ns)": 512175227061, "Completed samples per second": 5250.69, "Max latency (ns)": 603602375008, "Mean latency (ns)": 2045144864, "Min duration satisfied": "Yes", "Min latency (ns)": 4364349, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 5251.6, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 5251.6, "characteristics.scheduled_queries_per_second.normalized_per_core": 656.45, "characteristics.scheduled_queries_per_second.normalized_per_processor": 656.45, "ck_system": "A30x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 5250, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "6064d7a89aca4de9", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 28639096, "90.00 percentile latency (ns)": 46292136, "95.00 percentile latency (ns)": 48095423, "97.00 percentile latency (ns)": 49134445, "99.00 percentile latency (ns)": 51011351, "99.90 percentile latency (ns)": 54560019, "Completed samples per second": 11000.3, "Max latency (ns)": 80697752, "Mean latency (ns)": 29099776, "Min duration satisfied": "Yes", "Min latency (ns)": 5174979, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 11000.75, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 11000.75, "characteristics.scheduled_queries_per_second.normalized_per_core": 1375.09375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1375.09375, "ck_system": "A30x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "23426d9405035639", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 94322826, "90.00 percentile latency (ns)": 101580333, "95.00 percentile latency (ns)": 103650670, "97.00 percentile latency (ns)": 105054102, "99.00 percentile latency (ns)": 107939147, "99.90 percentile latency (ns)": 112938351, "Completed samples per second": 5192.33, "Max latency (ns)": 1064885693, "Mean latency (ns)": 94389113, "Min duration satisfied": "Yes", "Min latency (ns)": 73769649, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 5201.46, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 5201.46, "characteristics.scheduled_queries_per_second.normalized_per_core": 650.1825, "characteristics.scheduled_queries_per_second.normalized_per_processor": 650.1825, "ck_system": "A30x8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x8_TRT_Triton", "system_name": "Gigabyte G482-Z54 (8x A30, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 5200, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "c3656044245b57d0", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 49071218, "90.00 percentile latency (ns)": 70274920, "95.00 percentile latency (ns)": 76584678, "97.00 percentile latency (ns)": 80818011, "99.00 percentile latency (ns)": 88815349, "99.90 percentile latency (ns)": 101613979, "Completed samples per second": 17497.69, "Max latency (ns)": 147161725, "Mean latency (ns)": 50124965, "Min duration satisfied": "Yes", "Min latency (ns)": 5436083, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 17498.52, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 2308.8578333333344, "characteristics.power.normalized_per_core": 288.6072291666668, "characteristics.power.normalized_per_processor": 288.6072291666668, "characteristics.scheduled_queries_per_second": 17498.52, "characteristics.scheduled_queries_per_second.normalized_per_core": 2187.315, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2187.315, "ck_system": "A100-PCIex8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT_MaxQ", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 17500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "95620b4124d5c9fc", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 36684639, "90.00 percentile latency (ns)": 59016736, "95.00 percentile latency (ns)": 67159022, "97.00 percentile latency (ns)": 72719585, "99.00 percentile latency (ns)": 84412378, "99.90 percentile latency (ns)": 105878874, "Completed samples per second": 7503.57, "Max latency (ns)": 141656042, "Mean latency (ns)": 39296151, "Min duration satisfied": "Yes", "Min latency (ns)": 4201629, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 7504.02, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 2263.458901830283, "characteristics.power.normalized_per_core": 282.9323627287854, "characteristics.power.normalized_per_processor": 282.9323627287854, "characteristics.scheduled_queries_per_second": 7504.02, "characteristics.scheduled_queries_per_second.normalized_per_core": 938.0025, "characteristics.scheduled_queries_per_second.normalized_per_processor": 938.0025, "ck_system": "A100-PCIex8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT_MaxQ", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "6c4e6b831e57a4ad", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 56815233, "90.00 percentile latency (ns)": 63877083, "95.00 percentile latency (ns)": 65746934, "97.00 percentile latency (ns)": 67203884, "99.00 percentile latency (ns)": 98880555, "99.90 percentile latency (ns)": 544618554989, "Completed samples per second": 25789.64, "Max latency (ns)": 606584489643, "Mean latency (ns)": 3065111679, "Min duration satisfied": "Yes", "Min latency (ns)": 5291881, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 25791.81, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 25791.81, "characteristics.scheduled_queries_per_second.normalized_per_core": 3223.97625, "characteristics.scheduled_queries_per_second.normalized_per_processor": 3223.97625, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 25800, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "72e56314297b0b87", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 58799924, "90.00 percentile latency (ns)": 66627431, "95.00 percentile latency (ns)": 69258337, "97.00 percentile latency (ns)": 71584396, "99.00 percentile latency (ns)": 88504836, "99.90 percentile latency (ns)": 545703792377, "Completed samples per second": 13098.28, "Max latency (ns)": 606892799140, "Mean latency (ns)": 3077424365, "Min duration satisfied": "Yes", "Min latency (ns)": 5042354, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 13099.47, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 13099.47, "characteristics.scheduled_queries_per_second.normalized_per_core": 1637.43375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1637.43375, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 13100, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "466cb4fec960ca46", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 44895004, "90.00 percentile latency (ns)": 62409169, "95.00 percentile latency (ns)": 67345357, "97.00 percentile latency (ns)": 70572144, "99.00 percentile latency (ns)": 76577763, "99.90 percentile latency (ns)": 86691282, "Completed samples per second": 10201.93, "Max latency (ns)": 109233712, "Mean latency (ns)": 45455629, "Min duration satisfied": "Yes", "Min latency (ns)": 7171478, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10202.91, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 10202.91, "characteristics.scheduled_queries_per_second.normalized_per_core": 2550.7275, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2550.7275, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10200, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "a2e3340bab5b864f", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 45116187, "90.00 percentile latency (ns)": 74091884, "95.00 percentile latency (ns)": 84048782, "97.00 percentile latency (ns)": 90828047, "99.00 percentile latency (ns)": 104473902, "99.90 percentile latency (ns)": 127917024, "Completed samples per second": 4860.09, "Max latency (ns)": 160326613, "Mean latency (ns)": 48178310, "Min duration satisfied": "Yes", "Min latency (ns)": 4567331, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 4860.48, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 4860.48, "characteristics.scheduled_queries_per_second.normalized_per_core": 1215.12, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1215.12, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4860, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "52ae1af46fcad4c9", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43399605, "90.00 percentile latency (ns)": 65405764, "95.00 percentile latency (ns)": 72425955, "97.00 percentile latency (ns)": 77633912, "99.00 percentile latency (ns)": 104502627, "99.90 percentile latency (ns)": 639468565526, "Completed samples per second": 380.85, "Max latency (ns)": 713430495840, "Mean latency (ns)": 3414767754, "Min duration satisfied": "Yes", "Min latency (ns)": 4144423, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 380.87, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 380.87, "characteristics.scheduled_queries_per_second.normalized_per_core": 380.87, "characteristics.scheduled_queries_per_second.normalized_per_processor": 380.87, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "system_name": "NVIDIA DGX-A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, HeteroMultiUse)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 380, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "0334c038041d1314", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41756784, "90.00 percentile latency (ns)": 68837397, "95.00 percentile latency (ns)": 77358698, "97.00 percentile latency (ns)": 83544848, "99.00 percentile latency (ns)": 124818067, "99.90 percentile latency (ns)": 1523088211364, "Completed samples per second": 160.36, "Max latency (ns)": 1693043705445, "Mean latency (ns)": 8237650556, "Min duration satisfied": "Yes", "Min latency (ns)": 6751918, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 160.37, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 160.37, "characteristics.scheduled_queries_per_second.normalized_per_core": 160.37, "characteristics.scheduled_queries_per_second.normalized_per_processor": 160.37, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_HeteroMultiUse", "system_name": "NVIDIA DGX-A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, HeteroMultiUse)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 160, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "babc53383d3ea5ce", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 76377888, "90.00 percentile latency (ns)": 100676496, "95.00 percentile latency (ns)": 106157432, "97.00 percentile latency (ns)": 109260585, "99.00 percentile latency (ns)": 114593008, "99.90 percentile latency (ns)": 122845331, "Completed samples per second": 10201.3, "Max latency (ns)": 136300178, "Mean latency (ns)": 76134020, "Min duration satisfied": "Yes", "Min latency (ns)": 7210552, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10202.91, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 1301.9906666666668, "characteristics.power.normalized_per_core": 325.4976666666667, "characteristics.power.normalized_per_processor": 325.4976666666667, "characteristics.scheduled_queries_per_second": 10202.91, "characteristics.scheduled_queries_per_second.normalized_per_core": 2550.7275, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2550.7275, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10200, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "41098faadd7ff2df", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 45152078, "90.00 percentile latency (ns)": 75121176, "95.00 percentile latency (ns)": 85961246, "97.00 percentile latency (ns)": 93415000, "99.00 percentile latency (ns)": 107657341, "99.90 percentile latency (ns)": 133131174, "Completed samples per second": 4302.23, "Max latency (ns)": 164675061, "Mean latency (ns)": 48622273, "Min duration satisfied": "Yes", "Min latency (ns)": 4867384, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 4302.49, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 4, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 1295.0170000000026, "characteristics.power.normalized_per_core": 323.75425000000064, "characteristics.power.normalized_per_processor": 323.75425000000064, "characteristics.scheduled_queries_per_second": 4302.49, "characteristics.scheduled_queries_per_second.normalized_per_core": 1075.6225, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1075.6225, "ck_system": "DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "512 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 1, "host_storage_capacity": "10 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 4, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-Station-A100_A100-SXM-80GBx4_TRT_MaxQ", "system_name": "NVIDIA DGX Station A100 (4x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4300, "task": "NLP", "task2": "nlp", "total_cores": 64, "uid": "d068fb66cd0610e1", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 55570945, "90.00 percentile latency (ns)": 74545240, "95.00 percentile latency (ns)": 79501281, "97.00 percentile latency (ns)": 82634356, "99.00 percentile latency (ns)": 88532119, "99.90 percentile latency (ns)": 96601744, "Completed samples per second": 20791.43, "Max latency (ns)": 136206050, "Mean latency (ns)": 55644494, "Min duration satisfied": "Yes", "Min latency (ns)": 5723507, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 20792.35, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 20792.35, "characteristics.scheduled_queries_per_second.normalized_per_core": 2599.04375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2599.04375, "ck_system": "A100-PCIex8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 20800, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "1ba143f6a0a9532e", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 57568553, "90.00 percentile latency (ns)": 89537514, "95.00 percentile latency (ns)": 99391928, "97.00 percentile latency (ns)": 105957154, "99.00 percentile latency (ns)": 118816222, "99.90 percentile latency (ns)": 140868019, "Completed samples per second": 9603.49, "Max latency (ns)": 174381564, "Mean latency (ns)": 59913688, "Min duration satisfied": "Yes", "Min latency (ns)": 5355932, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 9604.47, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 9604.47, "characteristics.scheduled_queries_per_second.normalized_per_core": 1200.55875, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1200.55875, "ck_system": "A100-PCIex8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex8_TRT", "system_name": "Gigabyte G482-Z54 (8x A100-PCIe, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 9600, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "7cb572b3cd876ab6", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 20549636, "90.00 percentile latency (ns)": 26997419, "95.00 percentile latency (ns)": 29346066, "97.00 percentile latency (ns)": 31041342, "99.00 percentile latency (ns)": 35002567, "99.90 percentile latency (ns)": 41167038, "Completed samples per second": 22953.9, "Max latency (ns)": 1004244377, "Mean latency (ns)": 21559469, "Min duration satisfied": "Yes", "Min latency (ns)": 12805977, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 22992.32, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 22992.32, "characteristics.scheduled_queries_per_second.normalized_per_core": 2874.04, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2874.04, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 23000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "274f9b36b4400cbf", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 44775280, "90.00 percentile latency (ns)": 48221870, "95.00 percentile latency (ns)": 49268751, "97.00 percentile latency (ns)": 49957222, "99.00 percentile latency (ns)": 51223084, "99.90 percentile latency (ns)": 53655726, "Completed samples per second": 11187.13, "Max latency (ns)": 1005272497, "Mean latency (ns)": 44856080, "Min duration satisfied": "Yes", "Min latency (ns)": 34190752, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 11205.87, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 11205.87, "characteristics.scheduled_queries_per_second.normalized_per_core": 1400.73375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1400.73375, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_Triton", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB, TensorRT, Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11205, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "961fc2e584b66b18", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 47107630, "90.00 percentile latency (ns)": 60137285, "95.00 percentile latency (ns)": 63771573, "97.00 percentile latency (ns)": 66202154, "99.00 percentile latency (ns)": 73715099, "99.90 percentile latency (ns)": 544298864755, "Completed samples per second": 21490.5, "Max latency (ns)": 606415168395, "Mean latency (ns)": 3026256424, "Min duration satisfied": "Yes", "Min latency (ns)": 6413929, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 21491.71, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 3563.5158415841615, "characteristics.power.normalized_per_core": 445.4394801980202, "characteristics.power.normalized_per_processor": 445.4394801980202, "characteristics.scheduled_queries_per_second": 21491.71, "characteristics.scheduled_queries_per_second.normalized_per_core": 2686.46375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2686.46375, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 21500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "b22bb44f568c059e", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 30761475, "90.00 percentile latency (ns)": 43523217, "95.00 percentile latency (ns)": 47716513, "97.00 percentile latency (ns)": 50835315, "99.00 percentile latency (ns)": 63754706, "99.90 percentile latency (ns)": 545238876295, "Completed samples per second": 10001.9, "Max latency (ns)": 606238334331, "Mean latency (ns)": 3000090184, "Min duration satisfied": "Yes", "Min latency (ns)": 4960507, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 10002.74, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.power": 3506.393729372935, "characteristics.power.normalized_per_core": 438.2992161716169, "characteristics.power.normalized_per_processor": 438.2992161716169, "characteristics.scheduled_queries_per_second": 10002.74, "characteristics.scheduled_queries_per_second.normalized_per_core": 1250.3425, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1250.3425, "ck_system": "DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ", "system_name": "NVIDIA DGX-A100 (8x A100-SXM-80GB, MaxQ, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 10000, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "06fc58c004643f7a", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 45531016, "90.00 percentile latency (ns)": 58535310, "95.00 percentile latency (ns)": 61848399, "97.00 percentile latency (ns)": 64119728, "99.00 percentile latency (ns)": 71396863, "99.90 percentile latency (ns)": 516772526384, "Completed samples per second": 7203.19, "Max latency (ns)": 604881196440, "Mean latency (ns)": 2163143233, "Min duration satisfied": "Yes", "Min latency (ns)": 5635146, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 7203.81, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 7203.81, "characteristics.scheduled_queries_per_second.normalized_per_core": 900.47625, "characteristics.scheduled_queries_per_second.normalized_per_processor": 900.47625, "ck_system": "A10x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7200, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "784268813e1a23f0", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 29100583, "90.00 percentile latency (ns)": 41966693, "95.00 percentile latency (ns)": 46167680, "97.00 percentile latency (ns)": 49149046, "99.00 percentile latency (ns)": 57242124, "99.90 percentile latency (ns)": 520668868949, "Completed samples per second": 3121.14, "Max latency (ns)": 604308161482, "Mean latency (ns)": 2149042542, "Min duration satisfied": "Yes", "Min latency (ns)": 5931628, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 3121.31, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 3121.31, "characteristics.scheduled_queries_per_second.normalized_per_core": 390.16375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 390.16375, "ck_system": "A10x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x8_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x8_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x A10, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3120, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "5954645839dbf8c8", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 82912054, "90.00 percentile latency (ns)": 85184885, "95.00 percentile latency (ns)": 93594820, "97.00 percentile latency (ns)": 104398848, "99.00 percentile latency (ns)": 121986193, "99.90 percentile latency (ns)": 146503296, "Completed samples per second": 19.54, "Max latency (ns)": 214970252, "Mean latency (ns)": 84098918, "Min duration satisfied": "Yes", "Min latency (ns)": 74881716, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 19.54, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "N/A", "accelerator_memory_configuration": "", "accelerator_model_name": "N/A", "accelerator_on-chip_memories": "", "accelerators_per_node": 0, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.scheduled_queries_per_second": 19.54, "characteristics.scheduled_queries_per_second.normalized_per_core": 0.1744642857142857, "characteristics.scheduled_queries_per_second.normalized_per_processor": 4.885, "ck_system": "Triton_CPU_4S_8380Hx1", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "OpenVino 2021.2.200", "host_memory_capacity": "1536 GB", "host_memory_configuration": "6 slots / 32GB each / 3200 MT/s per socket", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8380H CPU @ 2.90GHz", "host_processors_per_node": 4, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "fp32", "management_firmware_version": "", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 270336, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 112, "normalize_processors": 4, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/Triton_CPU_4S_8380Hx1", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS", "other_hardware": "", "other_software_stack": "Tensorflow 2.4.0, OpenVino 2021.2.200, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 10833, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "No", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "The original weight filename: bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "CPU Inference on Triton Inference Server", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Triton_CPU_4S_8380Hx1", "system_name": "Supermicro SYS-240P-TNRT (Cooper Lake running Triton)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 19.5, "task": "NLP", "task2": "nlp", "total_cores": 112, "uid": "b72e2eb7c14b1991", "use_accelerator": false, "weight_data_types": "int8", "weight_transformations": "We transfer the weight from int8 datatype in ONNX file to int8 datatype in OpenVino IR file." }, { "50.00 percentile latency (ns)": 34853095, "90.00 percentile latency (ns)": 37383930, "95.00 percentile latency (ns)": 38068381, "97.00 percentile latency (ns)": 38518954, "99.00 percentile latency (ns)": 39424412, "99.90 percentile latency (ns)": 41269634, "Completed samples per second": 22091.16, "Max latency (ns)": 1004984393, "Mean latency (ns)": 34915150, "Min duration satisfied": "Yes", "Min latency (ns)": 27766200, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 22460.98, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 22460.98, "characteristics.scheduled_queries_per_second.normalized_per_core": 2807.6225, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2807.6225, "ck_system": "DGX-A100_A100-SXM4x8_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x8_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x8_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 22455, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "a70808a0a4dd8130", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 45513854, "90.00 percentile latency (ns)": 49051511, "95.00 percentile latency (ns)": 50175450, "97.00 percentile latency (ns)": 50912198, "99.00 percentile latency (ns)": 52464311, "99.90 percentile latency (ns)": 59259748, "Completed samples per second": 11018.76, "Max latency (ns)": 1029674966, "Mean latency (ns)": 45693170, "Min duration satisfied": "Yes", "Min latency (ns)": 35290297, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 11207.18, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 11207.18, "characteristics.scheduled_queries_per_second.normalized_per_core": 1400.8975, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1400.8975, "ck_system": "DGX-A100_A100-SXM4x8_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x8_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x8_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11205, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "5ab740cfb67ccdad", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 53687247, "90.00 percentile latency (ns)": 69911934, "95.00 percentile latency (ns)": 75133338, "97.00 percentile latency (ns)": 78873349, "99.00 percentile latency (ns)": 107922953, "99.90 percentile latency (ns)": 54134660262, "Completed samples per second": 7590.41, "Max latency (ns)": 61558069733, "Mean latency (ns)": 346375630, "Min duration satisfied": "Yes", "Min latency (ns)": 5263471, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 7596.02, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 20, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 7596.02, "characteristics.scheduled_queries_per_second.normalized_per_core": 379.80100000000004, "characteristics.scheduled_queries_per_second.normalized_per_processor": 379.80100000000004, "ck_system": "T4x20_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 20, "normalize_processors": 20, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x20_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x20_TRT", "system_name": "Supermicro 6049GP-TRT-OTO-29 (20x T4, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7600, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "786a514622a93e74", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 56717875, "90.00 percentile latency (ns)": 75989055, "95.00 percentile latency (ns)": 81407724, "97.00 percentile latency (ns)": 85015099, "99.00 percentile latency (ns)": 98669088, "99.90 percentile latency (ns)": 67685714492, "Completed samples per second": 3644.1, "Max latency (ns)": 75758550087, "Mean latency (ns)": 403849724, "Min duration satisfied": "Yes", "Min latency (ns)": 6120767, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 3646.73, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 20, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 3646.73, "characteristics.scheduled_queries_per_second.normalized_per_core": 182.3365, "characteristics.scheduled_queries_per_second.normalized_per_processor": 182.3365, "ck_system": "T4x20_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 20, "normalize_processors": 20, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x20_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x20_TRT", "system_name": "Supermicro 6049GP-TRT-OTO-29 (20x T4, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3650, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "a31e26c4691967db", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 59698693, "90.00 percentile latency (ns)": 86165096, "95.00 percentile latency (ns)": 93583030, "97.00 percentile latency (ns)": 98086467, "99.00 percentile latency (ns)": 105999420, "99.90 percentile latency (ns)": 115708021, "Completed samples per second": 5192.92, "Max latency (ns)": 125093955, "Mean latency (ns)": 61616990, "Min duration satisfied": "Yes", "Min latency (ns)": 3021996, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 5199.74, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 2, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 5199.74, "characteristics.scheduled_queries_per_second.normalized_per_core": 2599.87, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2599.87, "ck_system": "A100-PCIex2_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 2, "normalize_processors": 2, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex2_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex2_TRT", "system_name": "Gigabyte G482-Z52 (2x A100-PCIe, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 5200, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "ceeeeb6d6419ab76", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 49300773, "90.00 percentile latency (ns)": 74049569, "95.00 percentile latency (ns)": 82949915, "97.00 percentile latency (ns)": 89167144, "99.00 percentile latency (ns)": 101978789, "99.90 percentile latency (ns)": 127092856, "Completed samples per second": 2426.95, "Max latency (ns)": 144587930, "Mean latency (ns)": 52156353, "Min duration satisfied": "Yes", "Min latency (ns)": 4383874, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 2427.82, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 2, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 2427.82, "characteristics.scheduled_queries_per_second.normalized_per_core": 1213.91, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1213.91, "ck_system": "A100-PCIex2_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 2, "normalize_processors": 2, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex2_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex2_TRT", "system_name": "Gigabyte G482-Z52 (2x A100-PCIe, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 2430, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "a59eebde0ed15fdd", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 64826723, "90.00 percentile latency (ns)": 89223947, "95.00 percentile latency (ns)": 96236329, "97.00 percentile latency (ns)": 101977958, "99.00 percentile latency (ns)": 120027716, "99.90 percentile latency (ns)": 54201124574, "Completed samples per second": 24736.35, "Max latency (ns)": 60593440408, "Mean latency (ns)": 361058808, "Min duration satisfied": "Yes", "Min latency (ns)": 3713653, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 24757.49, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 24757.49, "characteristics.scheduled_queries_per_second.normalized_per_core": 3094.68625, "characteristics.scheduled_queries_per_second.normalized_per_processor": 3094.68625, "ck_system": "DGX-A100_A100-SXM4x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x8_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x8_TRT", "system_name": "NVIDIA DGX-A100 (8x A100-SXM4, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 24750, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "d80b1dc75374321a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 39529665, "90.00 percentile latency (ns)": 60082421, "95.00 percentile latency (ns)": 66899807, "97.00 percentile latency (ns)": 72123423, "99.00 percentile latency (ns)": 95957353, "99.90 percentile latency (ns)": 54442524388, "Completed samples per second": 11496.77, "Max latency (ns)": 60478459334, "Mean latency (ns)": 336435663, "Min duration satisfied": "Yes", "Min latency (ns)": 5188031, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 11502.44, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 11502.44, "characteristics.scheduled_queries_per_second.normalized_per_core": 1437.805, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1437.805, "ck_system": "DGX-A100_A100-SXM4x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x8_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x8_TRT", "system_name": "NVIDIA DGX-A100 (8x A100-SXM4, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 11500, "task": "NLP", "task2": "nlp", "total_cores": 128, "uid": "3d4cc2a84b9496a8", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 61165938, "90.00 percentile latency (ns)": 79801384, "95.00 percentile latency (ns)": 85224889, "97.00 percentile latency (ns)": 88919477, "99.00 percentile latency (ns)": 102074844, "99.90 percentile latency (ns)": 79833158470, "Completed samples per second": 3095.69, "Max latency (ns)": 88349116340, "Mean latency (ns)": 450118001, "Min duration satisfied": "Yes", "Min latency (ns)": 6050254, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 3097.22, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 3097.22, "characteristics.scheduled_queries_per_second.normalized_per_core": 387.1525, "characteristics.scheduled_queries_per_second.normalized_per_processor": 387.1525, "ck_system": "T4x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x8_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x8_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x T4, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3100, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "94fcbbfa0dcc5c20", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 50930530, "90.00 percentile latency (ns)": 70672103, "95.00 percentile latency (ns)": 76584881, "97.00 percentile latency (ns)": 80552829, "99.00 percentile latency (ns)": 91814037, "99.90 percentile latency (ns)": 172639318170, "Completed samples per second": 1418.47, "Max latency (ns)": 192436283051, "Mean latency (ns)": 876745204, "Min duration satisfied": "Yes", "Min latency (ns)": 6352234, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "BERT SERVER", "Scenario": "server", "Scheduled samples per second": 1418.73, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 1418.73, "characteristics.scheduled_queries_per_second.normalized_per_core": 177.34125, "characteristics.scheduled_queries_per_second.normalized_per_processor": 177.34125, "ck_system": "T4x8_TRT", "ck_used": false, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x8_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x8_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (8x T4, TensorRT)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 1420, "task": "NLP", "task2": "nlp", "total_cores": 56, "uid": "aff4d45858300804", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 38918458, "90.00 percentile latency (ns)": 43491239, "95.00 percentile latency (ns)": 44853735, "97.00 percentile latency (ns)": 45778533, "99.00 percentile latency (ns)": 47521716, "99.90 percentile latency (ns)": 50886446, "Completed samples per second": 6899.96, "Max latency (ns)": 1012342421, "Mean latency (ns)": 38592788, "Min duration satisfied": "Yes", "Min latency (ns)": 23550722, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 7016.31, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 20, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 7016.31, "characteristics.scheduled_queries_per_second.normalized_per_core": 350.81550000000004, "characteristics.scheduled_queries_per_second.normalized_per_processor": 350.81550000000004, "ck_system": "T4x20_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 20, "normalize_processors": 20, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x20_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x20_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 7020, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "647cf8a1a85f674c", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 38353514, "90.00 percentile latency (ns)": 45007623, "95.00 percentile latency (ns)": 46957795, "97.00 percentile latency (ns)": 48316886, "99.00 percentile latency (ns)": 51725707, "99.90 percentile latency (ns)": 58518709, "Completed samples per second": 3370.31, "Max latency (ns)": 62795076, "Mean latency (ns)": 37881346, "Min duration satisfied": "Yes", "Min latency (ns)": 20984030, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 3371.97, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 20, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 3371.97, "characteristics.scheduled_queries_per_second.normalized_per_core": 168.5985, "characteristics.scheduled_queries_per_second.normalized_per_processor": 168.5985, "ck_system": "T4x20_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 20, "normalize_processors": 20, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x20_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x20_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 3375, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "aa894fb6984bb47c", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 40137343, "90.00 percentile latency (ns)": 45326580, "95.00 percentile latency (ns)": 46908425, "97.00 percentile latency (ns)": 47968913, "99.00 percentile latency (ns)": 50087945, "99.90 percentile latency (ns)": 54647111, "Completed samples per second": 2786.47, "Max latency (ns)": 59856139, "Mean latency (ns)": 39966626, "Min duration satisfied": "Yes", "Min latency (ns)": 23979126, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 2787.5, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 2787.5, "characteristics.scheduled_queries_per_second.normalized_per_core": 348.4375, "characteristics.scheduled_queries_per_second.normalized_per_processor": 348.4375, "ck_system": "T4x8_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x8_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x8_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 2790, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "f1878494594c08a1", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 39723632, "90.00 percentile latency (ns)": 47455898, "95.00 percentile latency (ns)": 49869020, "97.00 percentile latency (ns)": 51596020, "99.00 percentile latency (ns)": 55066142, "99.90 percentile latency (ns)": 61362049, "Completed samples per second": 1330.55, "Max latency (ns)": 100313972, "Mean latency (ns)": 39544840, "Min duration satisfied": "Yes", "Min latency (ns)": 20617007, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 1330.8, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 8, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 1330.8, "characteristics.scheduled_queries_per_second.normalized_per_core": 166.35, "characteristics.scheduled_queries_per_second.normalized_per_processor": 166.35, "ck_system": "T4x8_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 8, "normalize_processors": 8, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x8_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x8_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 1332, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "c5d7f004ca59f47e", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 33229834, "90.00 percentile latency (ns)": 39334591, "95.00 percentile latency (ns)": 40583482, "97.00 percentile latency (ns)": 41360235, "99.00 percentile latency (ns)": 42723613, "99.90 percentile latency (ns)": 45041071, "Completed samples per second": 4384.27, "Max latency (ns)": 47786681, "Mean latency (ns)": 33281319, "Min duration satisfied": "Yes", "Min latency (ns)": 21913713, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 4386.07, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 2, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 4386.07, "characteristics.scheduled_queries_per_second.normalized_per_core": 2193.035, "characteristics.scheduled_queries_per_second.normalized_per_processor": 2193.035, "ck_system": "A100-PCIex2_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 2, "normalize_processors": 2, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex2_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex2_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 4390, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "454595e677924663", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 68019007, "90.00 percentile latency (ns)": 81488722, "95.00 percentile latency (ns)": 86053303, "97.00 percentile latency (ns)": 89473697, "99.00 percentile latency (ns)": 96605250, "99.90 percentile latency (ns)": 109939685, "Completed samples per second": 2366.84, "Max latency (ns)": 122560710, "Mean latency (ns)": 68352311, "Min duration satisfied": "Yes", "Min latency (ns)": 42137663, "Min queries satisfied": "Yes", "Mode": "Performance", "Performance constraints satisfied": "Yes", "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "server", "Scheduled samples per second": 2367.87, "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 2, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.scheduled_queries_per_second": 2367.87, "characteristics.scheduled_queries_per_second.normalized_per_core": 1183.935, "characteristics.scheduled_queries_per_second.normalized_per_processor": 1183.935, "ck_system": "A100-PCIex2_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "SQuAD v1.1", "dataset_link": "", "dim_x_default": "seq_number", "dim_y_default": "characteristics.scheduled_queries_per_second", "dim_y_maximize": false, "division": "closed", "formal_model": "bert", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "bert-99.9", "input_data_types": "int32", "max_async_queries": 0, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 270336, "mlperf_version": 0.7, "normalize_cores": 2, "normalize_processors": 2, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex2_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 10833, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "bert_large_v1_1_fake_quant.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex2_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "datacenter", "target_latency (ns)": 130000000, "target_qps": 2370, "task": "NLP", "task2": "nlp", "total_cores": 240, "uid": "aed1c49bda5cdcd1", "use_accelerator": true, "weight_data_types": "fp16", "weight_transformations": "quantization, affine fusion" } ]