[ { "50.00 percentile latency (ns)": 41538585, "90.00 percentile latency (ns)": 42005054, "90th percentile latency (ns)": 42005054, "95.00 percentile latency (ns)": 42105075, "97.00 percentile latency (ns)": 42178454, "99.00 percentile latency (ns)": 42336163, "99.90 percentile latency (ns)": 42584562, "Max latency (ns)": 45324360, "Mean latency (ns)": 41559596, "Min duration satisfied": "Yes", "Min latency (ns)": 40596235, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.04, "QPS w/o loadgen overhead": 24.06, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 42.005054, "characteristics.90th_percentile_latency_ns": 42005054.0, "characteristics.90th_percentile_latency_s": 0.042005054, "characteristics.90th_percentile_latency_us": 42005.054, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A30x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "7e21f4c318d5ad94", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41538585, "90.00 percentile latency (ns)": 42005054, "90th percentile latency (ns)": 42005054, "95.00 percentile latency (ns)": 42105075, "97.00 percentile latency (ns)": 42178454, "99.00 percentile latency (ns)": 42336163, "99.90 percentile latency (ns)": 42584562, "Max latency (ns)": 45324360, "Mean latency (ns)": 41559596, "Min duration satisfied": "Yes", "Min latency (ns)": 40596235, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 24.04, "QPS w/o loadgen overhead": 24.06, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 42.005054, "characteristics.90th_percentile_latency_ns": 42005054.0, "characteristics.90th_percentile_latency_s": 0.042005054, "characteristics.90th_percentile_latency_us": 42005.054, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A30x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "fc8045af8b587e4b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 884117079, "90.00 percentile latency (ns)": 884570991, "90th percentile latency (ns)": 884570991, "95.00 percentile latency (ns)": 884734997, "97.00 percentile latency (ns)": 884882871, "99.00 percentile latency (ns)": 885074323, "99.90 percentile latency (ns)": 885446839, "Max latency (ns)": 991128333, "Mean latency (ns)": 884199803, "Min duration satisfied": "Yes", "Min latency (ns)": 882893269, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.13, "QPS w/o loadgen overhead": 1.13, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 884.570991, "characteristics.90th_percentile_latency_ns": 884570991.0, "characteristics.90th_percentile_latency_s": 0.884570991, "characteristics.90th_percentile_latency_us": 884570.991, "characteristics.enhancing tumor": 0.7803, "characteristics.mean": 0.85443, "characteristics.power": 11.553131978838147, "characteristics.power.normalized_per_core": 11.553131978838147, "characteristics.power.normalized_per_processor": 11.553131978838147, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9133, "ck_system": "Xavier_NX_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Xavier_NX_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT_MaxQ", "system_name": "Auvidea JNX30 Xavier NX (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "c186a06c60c4e7a4", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 884117079, "90.00 percentile latency (ns)": 884570991, "90th percentile latency (ns)": 884570991, "95.00 percentile latency (ns)": 884734997, "97.00 percentile latency (ns)": 884882871, "99.00 percentile latency (ns)": 885074323, "99.90 percentile latency (ns)": 885446839, "Max latency (ns)": 991128333, "Mean latency (ns)": 884199803, "Min duration satisfied": "Yes", "Min latency (ns)": 882893269, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.13, "QPS w/o loadgen overhead": 1.13, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 884.570991, "characteristics.90th_percentile_latency_ns": 884570991.0, "characteristics.90th_percentile_latency_s": 0.884570991, "characteristics.90th_percentile_latency_us": 884570.991, "characteristics.enhancing tumor": 0.7803, "characteristics.mean": 0.85443, "characteristics.power": 11.553131978838147, "characteristics.power.normalized_per_core": 11.553131978838147, "characteristics.power.normalized_per_processor": 11.553131978838147, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9133, "ck_system": "Xavier_NX_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Xavier_NX_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT_MaxQ", "system_name": "Auvidea JNX30 Xavier NX (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "94873a95b120fc71", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 414179613, "90.00 percentile latency (ns)": 417710438, "90th percentile latency (ns)": 417710438, "95.00 percentile latency (ns)": 418741460, "97.00 percentile latency (ns)": 419581017, "99.00 percentile latency (ns)": 420558564, "99.90 percentile latency (ns)": 421895040, "Max latency (ns)": 422402218, "Mean latency (ns)": 414255140, "Min duration satisfied": "Yes", "Min latency (ns)": 406274794, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.41, "QPS w/o loadgen overhead": 2.41, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 417.710438, "characteristics.90th_percentile_latency_ns": 417710438.0, "characteristics.90th_percentile_latency_s": 0.417710438, "characteristics.90th_percentile_latency_us": 417710.438, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9142, "ck_system": "AGX_Xavier_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT_Triton", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "292149a3b4dbf4b4", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 414179613, "90.00 percentile latency (ns)": 417710438, "90th percentile latency (ns)": 417710438, "95.00 percentile latency (ns)": 418741460, "97.00 percentile latency (ns)": 419581017, "99.00 percentile latency (ns)": 420558564, "99.90 percentile latency (ns)": 421895040, "Max latency (ns)": 422402218, "Mean latency (ns)": 414255140, "Min duration satisfied": "Yes", "Min latency (ns)": 406274794, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.41, "QPS w/o loadgen overhead": 2.41, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 417.710438, "characteristics.90th_percentile_latency_ns": 417710438.0, "characteristics.90th_percentile_latency_s": 0.417710438, "characteristics.90th_percentile_latency_us": 417710.438, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9142, "ck_system": "AGX_Xavier_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT_Triton", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "484215b2315a2847", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26566164, "90.00 percentile latency (ns)": 27083671, "90th percentile latency (ns)": 27083671, "95.00 percentile latency (ns)": 27123290, "97.00 percentile latency (ns)": 27284320, "99.00 percentile latency (ns)": 27481039, "99.90 percentile latency (ns)": 28447494, "Max latency (ns)": 33978730, "Mean latency (ns)": 26559552, "Min duration satisfied": "Yes", "Min latency (ns)": 25521704, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.61, "QPS w/o loadgen overhead": 37.65, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 27.083671, "characteristics.90th_percentile_latency_ns": 27083671.0, "characteristics.90th_percentile_latency_s": 0.027083671, "characteristics.90th_percentile_latency_us": 27083.671, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIex1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "8e9c17ac0e4bda8b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26566164, "90.00 percentile latency (ns)": 27083671, "90th percentile latency (ns)": 27083671, "95.00 percentile latency (ns)": 27123290, "97.00 percentile latency (ns)": 27284320, "99.00 percentile latency (ns)": 27481039, "99.90 percentile latency (ns)": 28447494, "Max latency (ns)": 33978730, "Mean latency (ns)": 26559552, "Min duration satisfied": "Yes", "Min latency (ns)": 25521704, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.61, "QPS w/o loadgen overhead": 37.65, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 27.083671, "characteristics.90th_percentile_latency_ns": 27083671.0, "characteristics.90th_percentile_latency_s": 0.027083671, "characteristics.90th_percentile_latency_us": 27083.671, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIex1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "f43ed6e7c0a64b1e", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 374252130, "90.00 percentile latency (ns)": 377172739, "90th percentile latency (ns)": 377172739, "95.00 percentile latency (ns)": 378087692, "97.00 percentile latency (ns)": 378696742, "99.00 percentile latency (ns)": 379774422, "99.90 percentile latency (ns)": 380474804, "Max latency (ns)": 380706271, "Mean latency (ns)": 374317483, "Min duration satisfied": "Yes", "Min latency (ns)": 367540314, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.67, "QPS w/o loadgen overhead": 2.67, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 377.172739, "characteristics.90th_percentile_latency_ns": 377172739.0, "characteristics.90th_percentile_latency_s": 0.377172739, "characteristics.90th_percentile_latency_us": 377172.739, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9142, "ck_system": "AGX_Xavier_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "aa4799ac5f4f2d88", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 374252130, "90.00 percentile latency (ns)": 377172739, "90th percentile latency (ns)": 377172739, "95.00 percentile latency (ns)": 378087692, "97.00 percentile latency (ns)": 378696742, "99.00 percentile latency (ns)": 379774422, "99.90 percentile latency (ns)": 380474804, "Max latency (ns)": 380706271, "Mean latency (ns)": 374317483, "Min duration satisfied": "Yes", "Min latency (ns)": 367540314, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.67, "QPS w/o loadgen overhead": 2.67, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 377.172739, "characteristics.90th_percentile_latency_ns": 377172739.0, "characteristics.90th_percentile_latency_s": 0.377172739, "characteristics.90th_percentile_latency_us": 377172.739, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9142, "ck_system": "AGX_Xavier_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "e480ccd14298935f", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 129764362, "90.00 percentile latency (ns)": 129799379, "90th percentile latency (ns)": 129799379, "95.00 percentile latency (ns)": 129810960, "97.00 percentile latency (ns)": 129819486, "99.00 percentile latency (ns)": 129843803, "99.90 percentile latency (ns)": 132555381, "Max latency (ns)": 135430149, "Mean latency (ns)": 129771824, "Min duration satisfied": "Yes", "Min latency (ns)": 129679271, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.71, "QPS w/o loadgen overhead": 7.71, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 129.799379, "characteristics.90th_percentile_latency_ns": 129799379.0, "characteristics.90th_percentile_latency_s": 0.129799379, "characteristics.90th_percentile_latency_us": 129799.379, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "759938344b65902d", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 129764362, "90.00 percentile latency (ns)": 129799379, "90th percentile latency (ns)": 129799379, "95.00 percentile latency (ns)": 129810960, "97.00 percentile latency (ns)": 129819486, "99.00 percentile latency (ns)": 129843803, "99.90 percentile latency (ns)": 132555381, "Max latency (ns)": 135430149, "Mean latency (ns)": 129771824, "Min duration satisfied": "Yes", "Min latency (ns)": 129679271, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.71, "QPS w/o loadgen overhead": 7.71, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 129.799379, "characteristics.90th_percentile_latency_ns": 129799379.0, "characteristics.90th_percentile_latency_s": 0.129799379, "characteristics.90th_percentile_latency_us": 129799.379, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "8bd4702415581b49", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 25398350, "90.00 percentile latency (ns)": 25734118, "90th percentile latency (ns)": 25734118, "95.00 percentile latency (ns)": 25778678, "97.00 percentile latency (ns)": 25849837, "99.00 percentile latency (ns)": 25962666, "99.90 percentile latency (ns)": 26886339, "Max latency (ns)": 30393992, "Mean latency (ns)": 25449710, "Min duration satisfied": "Yes", "Min latency (ns)": 25065348, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.23, "QPS w/o loadgen overhead": 39.29, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 25.734118, "characteristics.90th_percentile_latency_ns": 25734118.0, "characteristics.90th_percentile_latency_s": 0.025734118, "characteristics.90th_percentile_latency_us": 25734.118, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIe-80GBx1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx1_TRT", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe-80GB, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "485aa099a68b30e9", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 25398350, "90.00 percentile latency (ns)": 25734118, "90th percentile latency (ns)": 25734118, "95.00 percentile latency (ns)": 25778678, "97.00 percentile latency (ns)": 25849837, "99.00 percentile latency (ns)": 25962666, "99.90 percentile latency (ns)": 26886339, "Max latency (ns)": 30393992, "Mean latency (ns)": 25449710, "Min duration satisfied": "Yes", "Min latency (ns)": 25065348, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.23, "QPS w/o loadgen overhead": 39.29, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 25.734118, "characteristics.90th_percentile_latency_ns": 25734118.0, "characteristics.90th_percentile_latency_s": 0.025734118, "characteristics.90th_percentile_latency_us": 25734.118, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIe-80GBx1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx1_TRT", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe-80GB, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "234cff8de4d68b3c", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 16657887, "90.00 percentile latency (ns)": 16672854, "90th percentile latency (ns)": 16672854, "95.00 percentile latency (ns)": 16677473, "97.00 percentile latency (ns)": 16680730, "99.00 percentile latency (ns)": 16688294, "99.90 percentile latency (ns)": 17673131, "Max latency (ns)": 19853219, "Mean latency (ns)": 16660950, "Min duration satisfied": "Yes", "Min latency (ns)": 16622290, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.0, "QPS w/o loadgen overhead": 60.02, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 16.672854, "characteristics.90th_percentile_latency_ns": 16672854.0, "characteristics.90th_percentile_latency_s": 0.016672854, "characteristics.90th_percentile_latency_us": 16672.854, "characteristics.enhancing tumor": 0.7789, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9142, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "d31dccd47c99f116", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 16657887, "90.00 percentile latency (ns)": 16672854, "90th percentile latency (ns)": 16672854, "95.00 percentile latency (ns)": 16677473, "97.00 percentile latency (ns)": 16680730, "99.00 percentile latency (ns)": 16688294, "99.90 percentile latency (ns)": 17673131, "Max latency (ns)": 19853219, "Mean latency (ns)": 16660950, "Min duration satisfied": "Yes", "Min latency (ns)": 16622290, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 60.0, "QPS w/o loadgen overhead": 60.02, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 16.672854, "characteristics.90th_percentile_latency_ns": 16672854.0, "characteristics.90th_percentile_latency_s": 0.016672854, "characteristics.90th_percentile_latency_us": 16672.854, "characteristics.enhancing tumor": 0.7789, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9142, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "27a5cbfcd627d8ba", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 651634855, "90.00 percentile latency (ns)": 652834093, "90th percentile latency (ns)": 652834093, "95.00 percentile latency (ns)": 653337743, "97.00 percentile latency (ns)": 653744527, "99.00 percentile latency (ns)": 675072455, "99.90 percentile latency (ns)": 738290751, "Max latency (ns)": 1090051147, "Mean latency (ns)": 652593626, "Min duration satisfied": "Yes", "Min latency (ns)": 649739087, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.53, "QPS w/o loadgen overhead": 1.53, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 652.834093, "characteristics.90th_percentile_latency_ns": 652834093.0, "characteristics.90th_percentile_latency_s": 0.652834093, "characteristics.90th_percentile_latency_us": 652834.093, "characteristics.enhancing tumor": 0.7803, "characteristics.mean": 0.85443, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9133, "ck_system": "Xavier_NX_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Xavier_NX_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT_Triton", "system_name": "NVIDIA Jetson Xavier NX (TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "fb3c2fdd76fa3448", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 651634855, "90.00 percentile latency (ns)": 652834093, "90th percentile latency (ns)": 652834093, "95.00 percentile latency (ns)": 653337743, "97.00 percentile latency (ns)": 653744527, "99.00 percentile latency (ns)": 675072455, "99.90 percentile latency (ns)": 738290751, "Max latency (ns)": 1090051147, "Mean latency (ns)": 652593626, "Min duration satisfied": "Yes", "Min latency (ns)": 649739087, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.53, "QPS w/o loadgen overhead": 1.53, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 652.834093, "characteristics.90th_percentile_latency_ns": 652834093.0, "characteristics.90th_percentile_latency_s": 0.652834093, "characteristics.90th_percentile_latency_us": 652834.093, "characteristics.enhancing tumor": 0.7803, "characteristics.mean": 0.85443, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9133, "ck_system": "Xavier_NX_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Xavier_NX_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT_Triton", "system_name": "NVIDIA Jetson Xavier NX (TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "0e7e4d56958f0a34", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 135511077, "90.00 percentile latency (ns)": 135604497, "90th percentile latency (ns)": 135604497, "95.00 percentile latency (ns)": 136037436, "97.00 percentile latency (ns)": 136102697, "99.00 percentile latency (ns)": 136246647, "99.90 percentile latency (ns)": 136339217, "Max latency (ns)": 136356357, "Mean latency (ns)": 135515108, "Min duration satisfied": "Yes", "Min latency (ns)": 135308066, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.38, "QPS w/o loadgen overhead": 7.38, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (1x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 135.604497, "characteristics.90th_percentile_latency_ns": 135604497.0, "characteristics.90th_percentile_latency_s": 0.135604497, "characteristics.90th_percentile_latency_us": 135604.497, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "A30-MIG_1x1g.6gb_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_1x1g.6gb_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_1x1g.6gb_TRT", "system_name": "Gigabyte G482-Z54 (1x A30-MIG-1x1g.6gb, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.40766, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "4497497d1dd36e7d", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 135511077, "90.00 percentile latency (ns)": 135604497, "90th percentile latency (ns)": 135604497, "95.00 percentile latency (ns)": 136037436, "97.00 percentile latency (ns)": 136102697, "99.00 percentile latency (ns)": 136246647, "99.90 percentile latency (ns)": 136339217, "Max latency (ns)": 136356357, "Mean latency (ns)": 135515108, "Min duration satisfied": "Yes", "Min latency (ns)": 135308066, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.38, "QPS w/o loadgen overhead": 7.38, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (1x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 135.604497, "characteristics.90th_percentile_latency_ns": 135604497.0, "characteristics.90th_percentile_latency_s": 0.135604497, "characteristics.90th_percentile_latency_us": 135604.497, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "A30-MIG_1x1g.6gb_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_1x1g.6gb_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_1x1g.6gb_TRT", "system_name": "Gigabyte G482-Z54 (1x A30-MIG-1x1g.6gb, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.40766, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "15a0e25493abccb4", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 470538899, "90.00 percentile latency (ns)": 472475601, "90th percentile latency (ns)": 472475601, "95.00 percentile latency (ns)": 473019246, "97.00 percentile latency (ns)": 473430339, "99.00 percentile latency (ns)": 474604161, "99.90 percentile latency (ns)": 476673596, "Max latency (ns)": 522795481, "Mean latency (ns)": 470769780, "Min duration satisfied": "Yes", "Min latency (ns)": 468405996, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.12, "QPS w/o loadgen overhead": 2.12, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 472.475601, "characteristics.90th_percentile_latency_ns": 472475601.0, "characteristics.90th_percentile_latency_s": 0.472475601, "characteristics.90th_percentile_latency_us": 472475.601, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85403, "characteristics.power": 11.66297074524347, "characteristics.power.normalized_per_core": 11.66297074524347, "characteristics.power.normalized_per_processor": 11.66297074524347, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9142, "ck_system": "AGX_Xavier_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT_MaxQ", "system_name": "Auvidea X220-LC AGX Xavier 32GB (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "9dc907f1b425f52a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 470538899, "90.00 percentile latency (ns)": 472475601, "90th percentile latency (ns)": 472475601, "95.00 percentile latency (ns)": 473019246, "97.00 percentile latency (ns)": 473430339, "99.00 percentile latency (ns)": 474604161, "99.90 percentile latency (ns)": 476673596, "Max latency (ns)": 522795481, "Mean latency (ns)": 470769780, "Min duration satisfied": "Yes", "Min latency (ns)": 468405996, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.12, "QPS w/o loadgen overhead": 2.12, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 472.475601, "characteristics.90th_percentile_latency_ns": 472475601.0, "characteristics.90th_percentile_latency_s": 0.472475601, "characteristics.90th_percentile_latency_us": 472475.601, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85403, "characteristics.power": 11.66297074524347, "characteristics.power.normalized_per_core": 11.66297074524347, "characteristics.power.normalized_per_processor": 11.66297074524347, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9142, "ck_system": "AGX_Xavier_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT_MaxQ", "system_name": "Auvidea X220-LC AGX Xavier 32GB (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "2dccd46499581e75", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 25171864, "90.00 percentile latency (ns)": 25462968, "90th percentile latency (ns)": 25462968, "95.00 percentile latency (ns)": 25476915, "97.00 percentile latency (ns)": 25482935, "99.00 percentile latency (ns)": 25493916, "99.90 percentile latency (ns)": 26095510, "Max latency (ns)": 29526173, "Mean latency (ns)": 25231508, "Min duration satisfied": "Yes", "Min latency (ns)": 24970280, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.59, "QPS w/o loadgen overhead": 39.63, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 25.462968, "characteristics.90th_percentile_latency_ns": 25462968.0, "characteristics.90th_percentile_latency_s": 0.025462968, "characteristics.90th_percentile_latency_us": 25462.968, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIe-80GBx1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe-80GB, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "acb4fdc9bc4425ad", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 25171864, "90.00 percentile latency (ns)": 25462968, "90th percentile latency (ns)": 25462968, "95.00 percentile latency (ns)": 25476915, "97.00 percentile latency (ns)": 25482935, "99.00 percentile latency (ns)": 25493916, "99.90 percentile latency (ns)": 26095510, "Max latency (ns)": 29526173, "Mean latency (ns)": 25231508, "Min duration satisfied": "Yes", "Min latency (ns)": 24970280, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 39.59, "QPS w/o loadgen overhead": 39.63, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 25.462968, "characteristics.90th_percentile_latency_ns": 25462968.0, "characteristics.90th_percentile_latency_s": 0.025462968, "characteristics.90th_percentile_latency_us": 25462.968, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIe-80GBx1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIe-80GBx1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIe-80GBx1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe-80GB, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "7f5bc4611a09f851", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 58709951, "90.00 percentile latency (ns)": 59260903, "90th percentile latency (ns)": 59260903, "95.00 percentile latency (ns)": 59398748, "97.00 percentile latency (ns)": 59559861, "99.00 percentile latency (ns)": 59734672, "99.90 percentile latency (ns)": 63890286, "Max latency (ns)": 68260125, "Mean latency (ns)": 58768611, "Min duration satisfied": "Yes", "Min latency (ns)": 55799183, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.0, "QPS w/o loadgen overhead": 17.02, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 59.260903, "characteristics.90th_percentile_latency_ns": 59260903.0, "characteristics.90th_percentile_latency_s": 0.059260903, "characteristics.90th_percentile_latency_us": 59260.903, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A10x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "bd74571ab063d733", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 58709951, "90.00 percentile latency (ns)": 59260903, "90th percentile latency (ns)": 59260903, "95.00 percentile latency (ns)": 59398748, "97.00 percentile latency (ns)": 59559861, "99.00 percentile latency (ns)": 59734672, "99.90 percentile latency (ns)": 63890286, "Max latency (ns)": 68260125, "Mean latency (ns)": 58768611, "Min duration satisfied": "Yes", "Min latency (ns)": 55799183, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 17.0, "QPS w/o loadgen overhead": 17.02, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 59.260903, "characteristics.90th_percentile_latency_ns": 59260903.0, "characteristics.90th_percentile_latency_s": 0.059260903, "characteristics.90th_percentile_latency_us": 59260.903, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A10x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "dbce2f4016b6d753", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 17024574, "90.00 percentile latency (ns)": 17041686, "90th percentile latency (ns)": 17041686, "95.00 percentile latency (ns)": 17046836, "97.00 percentile latency (ns)": 17050532, "99.00 percentile latency (ns)": 17060361, "99.90 percentile latency (ns)": 18238630, "Max latency (ns)": 21714409, "Mean latency (ns)": 17028729, "Min duration satisfied": "Yes", "Min latency (ns)": 16976715, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.71, "QPS w/o loadgen overhead": 58.72, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 17.041686, "characteristics.90th_percentile_latency_ns": 17041686.0, "characteristics.90th_percentile_latency_s": 0.017041686, "characteristics.90th_percentile_latency_us": 17041.686, "characteristics.enhancing tumor": 0.7789, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9142, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_Triton", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "45c44b5a617e3830", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 17024574, "90.00 percentile latency (ns)": 17041686, "90th percentile latency (ns)": 17041686, "95.00 percentile latency (ns)": 17046836, "97.00 percentile latency (ns)": 17050532, "99.00 percentile latency (ns)": 17060361, "99.90 percentile latency (ns)": 18238630, "Max latency (ns)": 21714409, "Mean latency (ns)": 17028729, "Min duration satisfied": "Yes", "Min latency (ns)": 16976715, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 58.71, "QPS w/o loadgen overhead": 58.72, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 17.041686, "characteristics.90th_percentile_latency_ns": 17041686.0, "characteristics.90th_percentile_latency_s": 0.017041686, "characteristics.90th_percentile_latency_us": 17041.686, "characteristics.enhancing tumor": 0.7789, "characteristics.mean": 0.85403, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9142, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_Triton", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "81abe99630a48fac", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 58844016, "90.00 percentile latency (ns)": 59403938, "90th percentile latency (ns)": 59403938, "95.00 percentile latency (ns)": 59544247, "97.00 percentile latency (ns)": 59722486, "99.00 percentile latency (ns)": 59885830, "99.90 percentile latency (ns)": 60091949, "Max latency (ns)": 61579179, "Mean latency (ns)": 58897031, "Min duration satisfied": "Yes", "Min latency (ns)": 55714095, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.96, "QPS w/o loadgen overhead": 16.98, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 59.403938, "characteristics.90th_percentile_latency_ns": 59403938.0, "characteristics.90th_percentile_latency_s": 0.059403938, "characteristics.90th_percentile_latency_us": 59403.938, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A10x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "200c467a49ca9051", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 58844016, "90.00 percentile latency (ns)": 59403938, "90th percentile latency (ns)": 59403938, "95.00 percentile latency (ns)": 59544247, "97.00 percentile latency (ns)": 59722486, "99.00 percentile latency (ns)": 59885830, "99.90 percentile latency (ns)": 60091949, "Max latency (ns)": 61579179, "Mean latency (ns)": 58897031, "Min duration satisfied": "Yes", "Min latency (ns)": 55714095, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.96, "QPS w/o loadgen overhead": 16.98, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 59.403938, "characteristics.90th_percentile_latency_ns": 59403938.0, "characteristics.90th_percentile_latency_s": 0.059403938, "characteristics.90th_percentile_latency_us": 59403.938, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A10x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A10x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "a44cd8df44350a40", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41628505, "90.00 percentile latency (ns)": 42050268, "90th percentile latency (ns)": 42050268, "95.00 percentile latency (ns)": 42151384, "97.00 percentile latency (ns)": 42216165, "99.00 percentile latency (ns)": 42366614, "99.90 percentile latency (ns)": 42583914, "Max latency (ns)": 43012534, "Mean latency (ns)": 41641805, "Min duration satisfied": "Yes", "Min latency (ns)": 40797752, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.99, "QPS w/o loadgen overhead": 24.01, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 42.050268, "characteristics.90th_percentile_latency_ns": 42050268.0, "characteristics.90th_percentile_latency_s": 0.042050268, "characteristics.90th_percentile_latency_us": 42050.268, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A30x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "5a9a11c4333e7f83", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 41628505, "90.00 percentile latency (ns)": 42050268, "90th percentile latency (ns)": 42050268, "95.00 percentile latency (ns)": 42151384, "97.00 percentile latency (ns)": 42216165, "99.00 percentile latency (ns)": 42366614, "99.90 percentile latency (ns)": 42583914, "Max latency (ns)": 43012534, "Mean latency (ns)": 41641805, "Min duration satisfied": "Yes", "Min latency (ns)": 40797752, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 23.99, "QPS w/o loadgen overhead": 24.01, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 42.050268, "characteristics.90th_percentile_latency_ns": 42050268.0, "characteristics.90th_percentile_latency_s": 0.042050268, "characteristics.90th_percentile_latency_us": 42050.268, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A30x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "93a9dea782a3afc7", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 128663704, "90.00 percentile latency (ns)": 128698460, "90th percentile latency (ns)": 128698460, "95.00 percentile latency (ns)": 128710954, "97.00 percentile latency (ns)": 128723187, "99.00 percentile latency (ns)": 128748955, "99.90 percentile latency (ns)": 128853082, "Max latency (ns)": 131206554, "Mean latency (ns)": 128666510, "Min duration satisfied": "Yes", "Min latency (ns)": 128588692, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.77, "QPS w/o loadgen overhead": 7.77, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 128.69846, "characteristics.90th_percentile_latency_ns": 128698460.0, "characteristics.90th_percentile_latency_s": 0.12869846, "characteristics.90th_percentile_latency_us": 128698.46, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "77d8ccf01f44aaa0", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 128663704, "90.00 percentile latency (ns)": 128698460, "90th percentile latency (ns)": 128698460, "95.00 percentile latency (ns)": 128710954, "97.00 percentile latency (ns)": 128723187, "99.00 percentile latency (ns)": 128748955, "99.90 percentile latency (ns)": 128853082, "Max latency (ns)": 131206554, "Mean latency (ns)": 128666510, "Min duration satisfied": "Yes", "Min latency (ns)": 128588692, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.77, "QPS w/o loadgen overhead": 7.77, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 128.69846, "characteristics.90th_percentile_latency_ns": 128698460.0, "characteristics.90th_percentile_latency_s": 0.12869846, "characteristics.90th_percentile_latency_us": 128698.46, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "system_name": "NVIDIA DGX A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 40, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "c7ee4fe6a3423305", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 135361459, "90.00 percentile latency (ns)": 135514017, "90th percentile latency (ns)": 135514017, "95.00 percentile latency (ns)": 135767877, "97.00 percentile latency (ns)": 135976797, "99.00 percentile latency (ns)": 136049057, "99.90 percentile latency (ns)": 137737748, "Max latency (ns)": 139502068, "Mean latency (ns)": 135409623, "Min duration satisfied": "Yes", "Min latency (ns)": 135157907, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.38, "QPS w/o loadgen overhead": 7.38, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (1x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 135.514017, "characteristics.90th_percentile_latency_ns": 135514017.0, "characteristics.90th_percentile_latency_s": 0.135514017, "characteristics.90th_percentile_latency_us": 135514.017, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "A30-MIG_1x1g.6gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_1x1g.6gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_1x1g.6gb_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A30-MIG-1x1g.6gb, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.40766, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "5a449509504f3e55", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 135361459, "90.00 percentile latency (ns)": 135514017, "90th percentile latency (ns)": 135514017, "95.00 percentile latency (ns)": 135767877, "97.00 percentile latency (ns)": 135976797, "99.00 percentile latency (ns)": 136049057, "99.90 percentile latency (ns)": 137737748, "Max latency (ns)": 139502068, "Mean latency (ns)": 135409623, "Min duration satisfied": "Yes", "Min latency (ns)": 135157907, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.38, "QPS w/o loadgen overhead": 7.38, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30 (1x1g.6gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 135.514017, "characteristics.90th_percentile_latency_ns": 135514017.0, "characteristics.90th_percentile_latency_s": 0.135514017, "characteristics.90th_percentile_latency_us": 135514.017, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.8544, "characteristics.tumor core": 0.8698, "characteristics.whole tumor": 0.9132, "ck_system": "A30-MIG_1x1g.6gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A30-MIG_1x1g.6gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0, Triton 21.07", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30-MIG_1x1g.6gb_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A30-MIG-1x1g.6gb, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 7.40766, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "80b3a4f75838a8bc", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26708996, "90.00 percentile latency (ns)": 27191390, "90th percentile latency (ns)": 27191390, "95.00 percentile latency (ns)": 27357015, "97.00 percentile latency (ns)": 27423312, "99.00 percentile latency (ns)": 27635113, "99.90 percentile latency (ns)": 27926349, "Max latency (ns)": 32673054, "Mean latency (ns)": 26676064, "Min duration satisfied": "Yes", "Min latency (ns)": 25687008, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.44, "QPS w/o loadgen overhead": 37.49, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 27.19139, "characteristics.90th_percentile_latency_ns": 27191390.0, "characteristics.90th_percentile_latency_s": 0.02719139, "characteristics.90th_percentile_latency_us": 27191.39, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIex1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "fa0e8a4d0ab588be", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 26708996, "90.00 percentile latency (ns)": 27191390, "90th percentile latency (ns)": 27191390, "95.00 percentile latency (ns)": 27357015, "97.00 percentile latency (ns)": 27423312, "99.00 percentile latency (ns)": 27635113, "99.90 percentile latency (ns)": 27926349, "Max latency (ns)": 32673054, "Mean latency (ns)": 26676064, "Min duration satisfied": "Yes", "Min latency (ns)": 25687008, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 37.44, "QPS w/o loadgen overhead": 37.49, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 27.19139, "characteristics.90th_percentile_latency_ns": 27191390.0, "characteristics.90th_percentile_latency_s": 0.02719139, "characteristics.90th_percentile_latency_us": 27191.39, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85437, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9132, "ck_system": "A100-PCIex1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 8.0.1, CUDA 11.3", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 20.04.4", "other_hardware": "", "other_software_stack": "TensorRT 8.0.1, CUDA 11.3, cuDNN 8.2.1, Driver 470.42.01, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "91a8f4e8712ebb47", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 627278614, "90.00 percentile latency (ns)": 627724593, "90th percentile latency (ns)": 627724593, "95.00 percentile latency (ns)": 627834657, "97.00 percentile latency (ns)": 627912812, "99.00 percentile latency (ns)": 628251692, "99.90 percentile latency (ns)": 633409519, "Max latency (ns)": 733824848, "Mean latency (ns)": 627407150, "Min duration satisfied": "Yes", "Min latency (ns)": 626261542, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.59, "QPS w/o loadgen overhead": 1.59, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 627.724593, "characteristics.90th_percentile_latency_ns": 627724593.0, "characteristics.90th_percentile_latency_s": 0.627724593, "characteristics.90th_percentile_latency_us": 627724.593, "characteristics.enhancing tumor": 0.7803, "characteristics.mean": 0.85443, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9133, "ck_system": "Xavier_NX_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Xavier_NX_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT", "system_name": "NVIDIA Jetson Xavier NX (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "2bab9cd244c4b08a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 627278614, "90.00 percentile latency (ns)": 627724593, "90th percentile latency (ns)": 627724593, "95.00 percentile latency (ns)": 627834657, "97.00 percentile latency (ns)": 627912812, "99.00 percentile latency (ns)": 628251692, "99.90 percentile latency (ns)": 633409519, "Max latency (ns)": 733824848, "Mean latency (ns)": 627407150, "Min duration satisfied": "Yes", "Min latency (ns)": 626261542, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.59, "QPS w/o loadgen overhead": 1.59, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 627.724593, "characteristics.90th_percentile_latency_ns": 627724593.0, "characteristics.90th_percentile_latency_s": 0.627724593, "characteristics.90th_percentile_latency_us": 627724.593, "characteristics.enhancing tumor": 0.7803, "characteristics.mean": 0.85443, "characteristics.tumor core": 0.8697, "characteristics.whole tumor": 0.9133, "ck_system": "Xavier_NX_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline scenario", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.1, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.1/tree/master/closed/NVIDIA/results/Xavier_NX_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04", "other_hardware": "", "other_software_stack": "JetPack 4.6, TensorRT 8.0.1, CUDA 10.2, cuDNN 8.2.3, DALI 0.31.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 1624344308455410291, "retraining": "No", "sample_index_rng_seed": 517984244576520566, "samples_per_query": 1, "schedule_rng_seed": 10051496985653635065, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT", "system_name": "NVIDIA Jetson Xavier NX (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "f7f02f2e432da6ea", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43559078, "90.00 percentile latency (ns)": 44000115, "90th percentile latency (ns)": 44000115, "95.00 percentile latency (ns)": 44077527, "97.00 percentile latency (ns)": 44119136, "99.00 percentile latency (ns)": 44339927, "99.90 percentile latency (ns)": 45384063, "Max latency (ns)": 52219111, "Mean latency (ns)": 43567327, "Min duration satisfied": "Yes", "Min latency (ns)": 42675549, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.94, "QPS w/o loadgen overhead": 22.95, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 44.000115, "characteristics.90th_percentile_latency_ns": 44000115.0, "characteristics.90th_percentile_latency_s": 0.044000115, "characteristics.90th_percentile_latency_us": 44000.115, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A30x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "bb0a15b6c64ea0b3", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43559078, "90.00 percentile latency (ns)": 44000115, "90th percentile latency (ns)": 44000115, "95.00 percentile latency (ns)": 44077527, "97.00 percentile latency (ns)": 44119136, "99.00 percentile latency (ns)": 44339927, "99.90 percentile latency (ns)": 45384063, "Max latency (ns)": 52219111, "Mean latency (ns)": 43567327, "Min duration satisfied": "Yes", "Min latency (ns)": 42675549, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.94, "QPS w/o loadgen overhead": 22.95, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 44.000115, "characteristics.90th_percentile_latency_ns": 44000115.0, "characteristics.90th_percentile_latency_s": 0.044000115, "characteristics.90th_percentile_latency_us": 44000.115, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A30x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "1a31504ae0851912", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 818536370, "90.00 percentile latency (ns)": 819081956, "90th percentile latency (ns)": 819081956, "95.00 percentile latency (ns)": 819264972, "97.00 percentile latency (ns)": 819397197, "99.00 percentile latency (ns)": 819628882, "99.90 percentile latency (ns)": 820308174, "Max latency (ns)": 884305309, "Mean latency (ns)": 818618121, "Min duration satisfied": "Yes", "Min latency (ns)": 817293056, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.22, "QPS w/o loadgen overhead": 1.22, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 819.081956, "characteristics.90th_percentile_latency_ns": 819081956.0, "characteristics.90th_percentile_latency_s": 0.819081956, "characteristics.90th_percentile_latency_us": 819081.956, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.power": 12.13694506089718, "characteristics.power.normalized_per_core": 12.13694506089718, "characteristics.power.normalized_per_processor": 12.13694506089718, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "Xavier_NX_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/Xavier_NX_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT_MaxQ", "system_name": "NVIDIA Jetson Xavier NX (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "122912885e8c831f", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 818536370, "90.00 percentile latency (ns)": 819081956, "90th percentile latency (ns)": 819081956, "95.00 percentile latency (ns)": 819264972, "97.00 percentile latency (ns)": 819397197, "99.00 percentile latency (ns)": 819628882, "99.90 percentile latency (ns)": 820308174, "Max latency (ns)": 884305309, "Mean latency (ns)": 818618121, "Min duration satisfied": "Yes", "Min latency (ns)": 817293056, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.22, "QPS w/o loadgen overhead": 1.22, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 819.081956, "characteristics.90th_percentile_latency_ns": 819081956.0, "characteristics.90th_percentile_latency_s": 0.819081956, "characteristics.90th_percentile_latency_us": 819081.956, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.power": 12.13694506089718, "characteristics.power.normalized_per_core": 12.13694506089718, "characteristics.power.normalized_per_processor": 12.13694506089718, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "Xavier_NX_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/Xavier_NX_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT_MaxQ", "system_name": "NVIDIA Jetson Xavier NX (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "3432da397948fd4a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 27792614, "90.00 percentile latency (ns)": 28492765, "90th percentile latency (ns)": 28492765, "95.00 percentile latency (ns)": 28585758, "97.00 percentile latency (ns)": 28628577, "99.00 percentile latency (ns)": 28741833, "99.90 percentile latency (ns)": 31616237, "Max latency (ns)": 42631698, "Mean latency (ns)": 27878417, "Min duration satisfied": "Yes", "Min latency (ns)": 27091268, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.81, "QPS w/o loadgen overhead": 35.87, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 28.492765, "characteristics.90th_percentile_latency_ns": 28492765.0, "characteristics.90th_percentile_latency_s": 0.028492765, "characteristics.90th_percentile_latency_us": 28492.765, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A100-PCIex1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "3543a59755bc2727", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 28084485, "90.00 percentile latency (ns)": 28881278, "90th percentile latency (ns)": 28881278, "95.00 percentile latency (ns)": 29038717, "97.00 percentile latency (ns)": 29103446, "99.00 percentile latency (ns)": 29217995, "99.90 percentile latency (ns)": 29463533, "Max latency (ns)": 32481339, "Mean latency (ns)": 28173875, "Min duration satisfied": "Yes", "Min latency (ns)": 27099393, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.45, "QPS w/o loadgen overhead": 35.49, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 28.881278, "characteristics.90th_percentile_latency_ns": 28881278.0, "characteristics.90th_percentile_latency_s": 0.028881278, "characteristics.90th_percentile_latency_us": 28881.278, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A100-PCIex1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT_Triton", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "5403cad0e8b83673", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 382185138, "90.00 percentile latency (ns)": 384838559, "90th percentile latency (ns)": 384838559, "95.00 percentile latency (ns)": 385673816, "97.00 percentile latency (ns)": 386329173, "99.00 percentile latency (ns)": 387008011, "99.90 percentile latency (ns)": 388997001, "Max latency (ns)": 389686572, "Mean latency (ns)": 382196844, "Min duration satisfied": "Yes", "Min latency (ns)": 375608933, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.62, "QPS w/o loadgen overhead": 2.62, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 384.838559, "characteristics.90th_percentile_latency_ns": 384838559.0, "characteristics.90th_percentile_latency_s": 0.384838559, "characteristics.90th_percentile_latency_us": 384838.559, "characteristics.enhancing tumor": 0.7796, "characteristics.mean": 0.8542, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.914, "ck_system": "AGX_Xavier_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "b0c243b4bd839d96", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 382185138, "90.00 percentile latency (ns)": 384838559, "90th percentile latency (ns)": 384838559, "95.00 percentile latency (ns)": 385673816, "97.00 percentile latency (ns)": 386329173, "99.00 percentile latency (ns)": 387008011, "99.90 percentile latency (ns)": 388997001, "Max latency (ns)": 389686572, "Mean latency (ns)": 382196844, "Min duration satisfied": "Yes", "Min latency (ns)": 375608933, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 2.62, "QPS w/o loadgen overhead": 2.62, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 384.838559, "characteristics.90th_percentile_latency_ns": 384838559.0, "characteristics.90th_percentile_latency_s": 0.384838559, "characteristics.90th_percentile_latency_us": 384838.559, "characteristics.enhancing tumor": 0.7796, "characteristics.mean": 0.8542, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.914, "ck_system": "AGX_Xavier_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "d2f05671ee0343aa", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 19843514, "90.00 percentile latency (ns)": 19919082, "90th percentile latency (ns)": 19919082, "95.00 percentile latency (ns)": 19931863, "97.00 percentile latency (ns)": 19940524, "99.00 percentile latency (ns)": 19957774, "99.90 percentile latency (ns)": 21043037, "Max latency (ns)": 23300796, "Mean latency (ns)": 19836893, "Min duration satisfied": "Yes", "Min latency (ns)": 19669469, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.38, "QPS w/o loadgen overhead": 50.41, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 19.919082, "characteristics.90th_percentile_latency_ns": 19919082.0, "characteristics.90th_percentile_latency_s": 0.019919082, "characteristics.90th_percentile_latency_us": 19919.082, "characteristics.enhancing tumor": 0.7798, "characteristics.mean": 0.8543, "characteristics.tumor core": 0.8691, "characteristics.whole tumor": 0.914, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_edge", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_edge", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_edge", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "be1faf4cdcba3f10", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 19843514, "90.00 percentile latency (ns)": 19919082, "90th percentile latency (ns)": 19919082, "95.00 percentile latency (ns)": 19931863, "97.00 percentile latency (ns)": 19940524, "99.00 percentile latency (ns)": 19957774, "99.90 percentile latency (ns)": 21043037, "Max latency (ns)": 23300796, "Mean latency (ns)": 19836893, "Min duration satisfied": "Yes", "Min latency (ns)": 19669469, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 50.38, "QPS w/o loadgen overhead": 50.41, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 19.919082, "characteristics.90th_percentile_latency_ns": 19919082.0, "characteristics.90th_percentile_latency_s": 0.019919082, "characteristics.90th_percentile_latency_us": 19919.082, "characteristics.enhancing tumor": 0.7798, "characteristics.mean": 0.8543, "characteristics.tumor core": 0.8691, "characteristics.whole tumor": 0.914, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_edge", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_edge", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_edge", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "f6b1092be934431b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 136246305, "90.00 percentile latency (ns)": 136277774, "90th percentile latency (ns)": 136277774, "95.00 percentile latency (ns)": 136291791, "97.00 percentile latency (ns)": 136312029, "99.00 percentile latency (ns)": 136389053, "99.90 percentile latency (ns)": 136484051, "Max latency (ns)": 136670180, "Mean latency (ns)": 136249422, "Min duration satisfied": "Yes", "Min latency (ns)": 136050979, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.34, "QPS w/o loadgen overhead": 7.34, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 136.277774, "characteristics.90th_percentile_latency_ns": 136277774.0, "characteristics.90th_percentile_latency_s": 0.136277774, "characteristics.90th_percentile_latency_us": 136277.774, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX-A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "38ce1bd4244a895c", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 136246305, "90.00 percentile latency (ns)": 136277774, "90th percentile latency (ns)": 136277774, "95.00 percentile latency (ns)": 136291791, "97.00 percentile latency (ns)": 136312029, "99.00 percentile latency (ns)": 136389053, "99.90 percentile latency (ns)": 136484051, "Max latency (ns)": 136670180, "Mean latency (ns)": 136249422, "Min duration satisfied": "Yes", "Min latency (ns)": 136050979, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.34, "QPS w/o loadgen overhead": 7.34, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 136.277774, "characteristics.90th_percentile_latency_ns": 136277774.0, "characteristics.90th_percentile_latency_s": 0.136277774, "characteristics.90th_percentile_latency_us": 136277.774, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT_Triton", "system_name": "NVIDIA DGX-A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "2103614d9fbb8e55", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 713168571, "90.00 percentile latency (ns)": 714337747, "90th percentile latency (ns)": 714337747, "95.00 percentile latency (ns)": 714652117, "97.00 percentile latency (ns)": 714832043, "99.00 percentile latency (ns)": 715203316, "99.90 percentile latency (ns)": 715940003, "Max latency (ns)": 820695210, "Mean latency (ns)": 713333138, "Min duration satisfied": "Yes", "Min latency (ns)": 711439033, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.4, "QPS w/o loadgen overhead": 1.4, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 714.337747, "characteristics.90th_percentile_latency_ns": 714337747.0, "characteristics.90th_percentile_latency_s": 0.714337747, "characteristics.90th_percentile_latency_us": 714337.747, "characteristics.enhancing tumor": 0.7796, "characteristics.mean": 0.8542, "characteristics.power": 13.363170549336717, "characteristics.power.normalized_per_core": 13.363170549336717, "characteristics.power.normalized_per_processor": 13.363170549336717, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.914, "ck_system": "AGX_Xavier_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT_MaxQ", "system_name": "NVIDIA Jetson AGX Xavier 32GB (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "fb57544760877d2e", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 713168571, "90.00 percentile latency (ns)": 714337747, "90th percentile latency (ns)": 714337747, "95.00 percentile latency (ns)": 714652117, "97.00 percentile latency (ns)": 714832043, "99.00 percentile latency (ns)": 715203316, "99.90 percentile latency (ns)": 715940003, "Max latency (ns)": 820695210, "Mean latency (ns)": 713333138, "Min duration satisfied": "Yes", "Min latency (ns)": 711439033, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.4, "QPS w/o loadgen overhead": 1.4, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 714.337747, "characteristics.90th_percentile_latency_ns": 714337747.0, "characteristics.90th_percentile_latency_s": 0.714337747, "characteristics.90th_percentile_latency_us": 714337.747, "characteristics.enhancing tumor": 0.7796, "characteristics.mean": 0.8542, "characteristics.power": 13.363170549336717, "characteristics.power.normalized_per_core": 13.363170549336717, "characteristics.power.normalized_per_processor": 13.363170549336717, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.914, "ck_system": "AGX_Xavier_TRT_MaxQ", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "32 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT_MaxQ", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT_MaxQ", "system_name": "NVIDIA Jetson AGX Xavier 32GB (MaxQ, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "d77a90fd24bbd89f", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 62233735, "90.00 percentile latency (ns)": 62723911, "90th percentile latency (ns)": 62723911, "95.00 percentile latency (ns)": 62832708, "97.00 percentile latency (ns)": 62897615, "99.00 percentile latency (ns)": 63014963, "99.90 percentile latency (ns)": 64760907, "Max latency (ns)": 115958102, "Mean latency (ns)": 62220716, "Min duration satisfied": "Yes", "Min latency (ns)": 58134508, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.05, "QPS w/o loadgen overhead": 16.07, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 62.723911, "characteristics.90th_percentile_latency_ns": 62723911.0, "characteristics.90th_percentile_latency_s": 0.062723911, "characteristics.90th_percentile_latency_us": 62723.911, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85383, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "A10x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "b8a9cd140c22f6c3", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 62233735, "90.00 percentile latency (ns)": 62723911, "90th percentile latency (ns)": 62723911, "95.00 percentile latency (ns)": 62832708, "97.00 percentile latency (ns)": 62897615, "99.00 percentile latency (ns)": 63014963, "99.90 percentile latency (ns)": 64760907, "Max latency (ns)": 115958102, "Mean latency (ns)": 62220716, "Min duration satisfied": "Yes", "Min latency (ns)": 58134508, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.05, "QPS w/o loadgen overhead": 16.07, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 62.723911, "characteristics.90th_percentile_latency_ns": 62723911.0, "characteristics.90th_percentile_latency_s": 0.062723911, "characteristics.90th_percentile_latency_us": 62723.911, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85383, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "A10x1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x1_TRT_Triton", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT_Triton", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "9228d3f094857bcb", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 20055412, "90.00 percentile latency (ns)": 20199843, "90th percentile latency (ns)": 20199843, "95.00 percentile latency (ns)": 20261862, "97.00 percentile latency (ns)": 20277298, "99.00 percentile latency (ns)": 20323113, "99.90 percentile latency (ns)": 21230212, "Max latency (ns)": 31399172, "Mean latency (ns)": 20089212, "Min duration satisfied": "Yes", "Min latency (ns)": 19975234, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 49.74, "QPS w/o loadgen overhead": 49.78, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 20.199843, "characteristics.90th_percentile_latency_ns": 20199843.0, "characteristics.90th_percentile_latency_s": 0.020199843, "characteristics.90th_percentile_latency_us": 20199.843, "characteristics.enhancing tumor": 0.7798, "characteristics.mean": 0.8543, "characteristics.tumor core": 0.8691, "characteristics.whole tumor": 0.914, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_Triton_edge", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_Triton_edge", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_Triton_edge", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "4a49cda362cec06c", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 20055412, "90.00 percentile latency (ns)": 20199843, "90th percentile latency (ns)": 20199843, "95.00 percentile latency (ns)": 20261862, "97.00 percentile latency (ns)": 20277298, "99.00 percentile latency (ns)": 20323113, "99.90 percentile latency (ns)": 21230212, "Max latency (ns)": 31399172, "Mean latency (ns)": 20089212, "Min duration satisfied": "Yes", "Min latency (ns)": 19975234, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 49.74, "QPS w/o loadgen overhead": 49.78, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 20.199843, "characteristics.90th_percentile_latency_ns": 20199843.0, "characteristics.90th_percentile_latency_s": 0.020199843, "characteristics.90th_percentile_latency_us": 20199.843, "characteristics.enhancing tumor": 0.7798, "characteristics.mean": 0.8543, "characteristics.tumor core": 0.8691, "characteristics.whole tumor": 0.914, "ck_system": "DGX-A100_A100-SXM-80GBx1_TRT_Triton_edge", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx1_TRT_Triton_edge", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0, Triton 21.02; GCC 7.5.0; Python 3.7.10", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GBx1_TRT_Triton_edge", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "edefa528f4fa4808", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 61898078, "90.00 percentile latency (ns)": 62378474, "90th percentile latency (ns)": 62378474, "95.00 percentile latency (ns)": 62475239, "97.00 percentile latency (ns)": 62533577, "99.00 percentile latency (ns)": 62613189, "99.90 percentile latency (ns)": 62840370, "Max latency (ns)": 64737950, "Mean latency (ns)": 61857072, "Min duration satisfied": "Yes", "Min latency (ns)": 57967433, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.15, "QPS w/o loadgen overhead": 16.17, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 62.378474, "characteristics.90th_percentile_latency_ns": 62378474.0, "characteristics.90th_percentile_latency_s": 0.062378474, "characteristics.90th_percentile_latency_us": 62378.474, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85383, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "A10x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "eb75455a88b373e5", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 61898078, "90.00 percentile latency (ns)": 62378474, "90th percentile latency (ns)": 62378474, "95.00 percentile latency (ns)": 62475239, "97.00 percentile latency (ns)": 62533577, "99.00 percentile latency (ns)": 62613189, "99.90 percentile latency (ns)": 62840370, "Max latency (ns)": 64737950, "Mean latency (ns)": 61857072, "Min duration satisfied": "Yes", "Min latency (ns)": 57967433, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 16.15, "QPS w/o loadgen overhead": 16.17, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA A10", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 62.378474, "characteristics.90th_percentile_latency_ns": 62378474.0, "characteristics.90th_percentile_latency_s": 0.062378474, "characteristics.90th_percentile_latency_us": 62378.474, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85383, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "A10x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A10x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A10x1_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x A10, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 14.7059, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "4c7ba444287abf89", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43716517, "90.00 percentile latency (ns)": 44112975, "90th percentile latency (ns)": 44112975, "95.00 percentile latency (ns)": 44200935, "97.00 percentile latency (ns)": 44276925, "99.00 percentile latency (ns)": 44409268, "99.90 percentile latency (ns)": 44637118, "Max latency (ns)": 49664402, "Mean latency (ns)": 43688222, "Min duration satisfied": "Yes", "Min latency (ns)": 42852210, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.87, "QPS w/o loadgen overhead": 22.89, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 44.112975, "characteristics.90th_percentile_latency_ns": 44112975.0, "characteristics.90th_percentile_latency_s": 0.044112975, "characteristics.90th_percentile_latency_us": 44112.975, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A30x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "384eb9b59840323a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43716517, "90.00 percentile latency (ns)": 44112975, "90th percentile latency (ns)": 44112975, "95.00 percentile latency (ns)": 44200935, "97.00 percentile latency (ns)": 44276925, "99.00 percentile latency (ns)": 44409268, "99.90 percentile latency (ns)": 44637118, "Max latency (ns)": 49664402, "Mean latency (ns)": 43688222, "Min duration satisfied": "Yes", "Min latency (ns)": 42852210, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 22.87, "QPS w/o loadgen overhead": 22.89, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "24 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A30", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 44.112975, "characteristics.90th_percentile_latency_ns": 44112975.0, "characteristics.90th_percentile_latency_s": 0.044112975, "characteristics.90th_percentile_latency_us": 44112.975, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A30x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A30x1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.46, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "preview", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A30x1_TRT", "system_name": "Gigabyte G482-Z54 (1x A30, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 19.6162, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "68bcdf063ae40087", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 135629348, "90.00 percentile latency (ns)": 135771445, "90th percentile latency (ns)": 135771445, "95.00 percentile latency (ns)": 135784980, "97.00 percentile latency (ns)": 135796933, "99.00 percentile latency (ns)": 135819715, "99.90 percentile latency (ns)": 135884036, "Max latency (ns)": 135927768, "Mean latency (ns)": 135646088, "Min duration satisfied": "Yes", "Min latency (ns)": 135451555, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.37, "QPS w/o loadgen overhead": 7.37, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 135.771445, "characteristics.90th_percentile_latency_ns": 135771445.0, "characteristics.90th_percentile_latency_s": 0.135771445, "characteristics.90th_percentile_latency_us": 135771.445, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "system_name": "NVIDIA DGX-A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "07a7645c2a2f5070", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 135629348, "90.00 percentile latency (ns)": 135771445, "90th percentile latency (ns)": 135771445, "95.00 percentile latency (ns)": 135784980, "97.00 percentile latency (ns)": 135796933, "99.00 percentile latency (ns)": 135819715, "99.90 percentile latency (ns)": 135884036, "Max latency (ns)": 135927768, "Mean latency (ns)": 135646088, "Min duration satisfied": "Yes", "Min latency (ns)": 135451555, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 7.37, "QPS w/o loadgen overhead": 7.37, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "80 GB", "accelerator_memory_configuration": "HBM2e", "accelerator_model_name": "NVIDIA A100-SXM-80GB (1x1g.10gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 135.771445, "characteristics.90th_percentile_latency_ns": 135771445.0, "characteristics.90th_percentile_latency_s": 0.135771445, "characteristics.90th_percentile_latency_us": 135771.445, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "2 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM-80GB-MIG_1x1g.10gb_TRT", "system_name": "NVIDIA DGX-A100 (1x A100-SXM-80GB-MIG-1x1g.10gb, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "0234ea091916eed3", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 27782584, "90.00 percentile latency (ns)": 28501488, "90th percentile latency (ns)": 28501488, "95.00 percentile latency (ns)": 28614357, "97.00 percentile latency (ns)": 28675047, "99.00 percentile latency (ns)": 28756636, "99.90 percentile latency (ns)": 32487245, "Max latency (ns)": 39944405, "Mean latency (ns)": 27895855, "Min duration satisfied": "Yes", "Min latency (ns)": 27039893, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.78, "QPS w/o loadgen overhead": 35.85, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 28.501488, "characteristics.90th_percentile_latency_ns": 28501488.0, "characteristics.90th_percentile_latency_s": 0.028501488, "characteristics.90th_percentile_latency_us": 28501.488, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A100-PCIex1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "d470747c284f6904", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 27782584, "90.00 percentile latency (ns)": 28501488, "90th percentile latency (ns)": 28501488, "95.00 percentile latency (ns)": 28614357, "97.00 percentile latency (ns)": 28675047, "99.00 percentile latency (ns)": 28756636, "99.90 percentile latency (ns)": 32487245, "Max latency (ns)": 39944405, "Mean latency (ns)": 27895855, "Min duration satisfied": "Yes", "Min latency (ns)": 27039893, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 35.78, "QPS w/o loadgen overhead": 35.85, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40 GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe-40GB", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 28.501488, "characteristics.90th_percentile_latency_ns": 28501488.0, "characteristics.90th_percentile_latency_s": 0.028501488, "characteristics.90th_percentile_latency_us": 28501.488, "characteristics.enhancing tumor": 0.7802, "characteristics.mean": 0.85387, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.9131, "ck_system": "A100-PCIex1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2.3, CUDA 11.1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "TensorRT 7.2.3, CUDA 11.1, cuDNN 8.1.1, Driver 460.32.03, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT", "system_name": "Gigabyte G482-Z54 (1x A100-PCIe, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "e98076ce184e5420", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 679158287, "90.00 percentile latency (ns)": 679654667, "90th percentile latency (ns)": 679654667, "95.00 percentile latency (ns)": 679824060, "97.00 percentile latency (ns)": 679922762, "99.00 percentile latency (ns)": 680201662, "99.90 percentile latency (ns)": 682561927, "Max latency (ns)": 686079164, "Mean latency (ns)": 679155258, "Min duration satisfied": "Yes", "Min latency (ns)": 677691834, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.47, "QPS w/o loadgen overhead": 1.47, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 679.654667, "characteristics.90th_percentile_latency_ns": 679654667.0, "characteristics.90th_percentile_latency_s": 0.679654667, "characteristics.90th_percentile_latency_us": 679654.667, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "Xavier_NX_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/Xavier_NX_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT", "system_name": "NVIDIA Jetson Xavier NX (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "47caf94e74277b3e", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 679158287, "90.00 percentile latency (ns)": 679654667, "90th percentile latency (ns)": 679654667, "95.00 percentile latency (ns)": 679824060, "97.00 percentile latency (ns)": 679922762, "99.00 percentile latency (ns)": 680201662, "99.90 percentile latency (ns)": 682561927, "Max latency (ns)": 686079164, "Mean latency (ns)": 679155258, "Min duration satisfied": "Yes", "Min latency (ns)": 677691834, "Min queries satisfied": "Yes", "Mode": "PerformanceOnly", "QPS w/ loadgen overhead": 1.47, "QPS w/o loadgen overhead": 1.47, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "boot_firmware_version": "", "characteristics.90th_percentile_latency_ms": 679.654667, "characteristics.90th_percentile_latency_ns": 679654667.0, "characteristics.90th_percentile_latency_s": 0.679654667, "characteristics.90th_percentile_latency_us": 679654.667, "characteristics.enhancing tumor": 0.78, "characteristics.mean": 0.85377, "characteristics.tumor core": 0.8683, "characteristics.whole tumor": 0.913, "ck_system": "Xavier_NX_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "disk_controllers": "", "disk_drives": "", "division": "closed", "filesystem": "", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2", "host_memory_capacity": "8 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32 GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "management_firmware_version": "", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 600000, "min_query_count": 1024, "mlperf_version": 1.0, "network_speed_mbit": "", "nics_enabled_connected": "", "nics_enabled_firmware": "", "nics_enabled_os": "", "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v1.0/tree/master/closed/NVIDIA/results/Xavier_NX_TRT", "number_of_nodes": 1, "number_of_type_nics_installed": "", "operating_system": "Ubuntu 18.04.4", "other_hardware": "", "other_software_stack": "21.03 Jetson CUDA-X AI Developer Preview, TensorRT 7.2.3, CUDA 10.2, cuDNN 8.0.0, DALI 0.30.0", "performance_issue_same": 0, "performance_issue_same_index": 0, "performance_issue_unique": 0, "performance_sample_count": 16, "power_management": "", "power_supply_details": "", "power_supply_quantity_and_rating_watts": "", "print_timestamps": 0, "problem": false, "qsl_rng_seed": 7322528924094909334, "retraining": "N", "sample_index_rng_seed": 1570999273408051088, "samples_per_query": 1, "schedule_rng_seed": 3507442325620259414, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT", "system_name": "NVIDIA Jetson Xavier NX (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "49ecd415a966cde3", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 27299333, "90.00 percentile latency (ns)": 27421021, "90th percentile latency (ns)": 27421021, "95.00 percentile latency (ns)": 27432583, "97.00 percentile latency (ns)": 27440858, "99.00 percentile latency (ns)": 27463931, "99.90 percentile latency (ns)": 33593240, "Max latency (ns)": 33990175, "Mean latency (ns)": 27313699, "Min duration satisfied": "Yes", "Min latency (ns)": 27145705, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 36.6, "QPS w/o loadgen overhead": 36.61, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 27.421021, "characteristics.90th_percentile_latency_ns": 27421021.0, "characteristics.90th_percentile_latency_s": 0.027421021, "characteristics.90th_percentile_latency_us": 27421.021, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.8538, "characteristics.tumor core": 0.8693, "characteristics.whole tumor": 0.9126, "ck_system": "DGX-A100_A100-SXM4x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1_TRT", "system_name": "NVIDIA DGX-A100 (1x A100-SXM4, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "c49d4ab8eb143ee6", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 27299333, "90.00 percentile latency (ns)": 27421021, "90th percentile latency (ns)": 27421021, "95.00 percentile latency (ns)": 27432583, "97.00 percentile latency (ns)": 27440858, "99.00 percentile latency (ns)": 27463931, "99.90 percentile latency (ns)": 33593240, "Max latency (ns)": 33990175, "Mean latency (ns)": 27313699, "Min duration satisfied": "Yes", "Min latency (ns)": 27145705, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 36.6, "QPS w/o loadgen overhead": 36.61, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 27.421021, "characteristics.90th_percentile_latency_ns": 27421021.0, "characteristics.90th_percentile_latency_s": 0.027421021, "characteristics.90th_percentile_latency_us": 27421.021, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.8538, "characteristics.tumor core": 0.8693, "characteristics.whole tumor": 0.9126, "ck_system": "DGX-A100_A100-SXM4x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1_TRT", "system_name": "NVIDIA DGX-A100 (1x A100-SXM4, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "f5c27eb6d4102b0a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 202865518, "90.00 percentile latency (ns)": 203000661, "90th percentile latency (ns)": 203000661, "95.00 percentile latency (ns)": 203043333, "97.00 percentile latency (ns)": 203080279, "99.00 percentile latency (ns)": 203147188, "99.90 percentile latency (ns)": 204034940, "Max latency (ns)": 204231962, "Mean latency (ns)": 202865337, "Min duration satisfied": "Yes", "Min latency (ns)": 202563943, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 4.93, "QPS w/o loadgen overhead": 4.93, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "5GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4 (1x1g.5gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 203.000661, "characteristics.90th_percentile_latency_ns": 203000661.0, "characteristics.90th_percentile_latency_s": 0.203000661, "characteristics.90th_percentile_latency_us": 203000.661, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9125, "ck_system": "DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "f81c4f1a877279a2", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 202865518, "90.00 percentile latency (ns)": 203000661, "90th percentile latency (ns)": 203000661, "95.00 percentile latency (ns)": 203043333, "97.00 percentile latency (ns)": 203080279, "99.00 percentile latency (ns)": 203147188, "99.90 percentile latency (ns)": 204034940, "Max latency (ns)": 204231962, "Mean latency (ns)": 202865337, "Min duration satisfied": "Yes", "Min latency (ns)": 202563943, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 4.93, "QPS w/o loadgen overhead": 4.93, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "5GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4 (1x1g.5gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 203.000661, "characteristics.90th_percentile_latency_ns": 203000661.0, "characteristics.90th_percentile_latency_s": 0.203000661, "characteristics.90th_percentile_latency_us": 203000.661, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9125, "ck_system": "DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "b886e4219a04d324", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43566469, "90.00 percentile latency (ns)": 44937372, "90th percentile latency (ns)": 44937372, "95.00 percentile latency (ns)": 45117983, "97.00 percentile latency (ns)": 45291245, "99.00 percentile latency (ns)": 45583413, "99.90 percentile latency (ns)": 49570775, "Max latency (ns)": 53543486, "Mean latency (ns)": 43704495, "Min duration satisfied": "Yes", "Min latency (ns)": 42045842, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 22.85, "QPS w/o loadgen overhead": 22.88, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 44.937372, "characteristics.90th_percentile_latency_ns": 44937372.0, "characteristics.90th_percentile_latency_s": 0.044937372, "characteristics.90th_percentile_latency_us": 44937.372, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85347, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9125, "ck_system": "A100-PCIex1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT_Triton", "system_name": "Gigabyte G482-Z52 (1x A100-PCIe, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "6602b2d6486cb31b", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 43566469, "90.00 percentile latency (ns)": 44937372, "90th percentile latency (ns)": 44937372, "95.00 percentile latency (ns)": 45117983, "97.00 percentile latency (ns)": 45291245, "99.00 percentile latency (ns)": 45583413, "99.90 percentile latency (ns)": 49570775, "Max latency (ns)": 53543486, "Mean latency (ns)": 43704495, "Min duration satisfied": "Yes", "Min latency (ns)": 42045842, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 22.85, "QPS w/o loadgen overhead": 22.88, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 44.937372, "characteristics.90th_percentile_latency_ns": 44937372.0, "characteristics.90th_percentile_latency_s": 0.044937372, "characteristics.90th_percentile_latency_us": 44937.372, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85347, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9125, "ck_system": "A100-PCIex1_TRT_Triton", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT_Triton", "system_name": "Gigabyte G482-Z52 (1x A100-PCIe, TensorRT, Triton)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "e9cb12a06c96da08", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 436391607, "90.00 percentile latency (ns)": 439273668, "90th percentile latency (ns)": 439273668, "95.00 percentile latency (ns)": 440216998, "97.00 percentile latency (ns)": 440882977, "99.00 percentile latency (ns)": 441936306, "99.90 percentile latency (ns)": 442847339, "Max latency (ns)": 443015405, "Mean latency (ns)": 436642629, "Min duration satisfied": "Yes", "Min latency (ns)": 432910202, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 2.29, "QPS w/o loadgen overhead": 2.29, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 439.273668, "characteristics.90th_percentile_latency_ns": 439273668.0, "characteristics.90th_percentile_latency_s": 0.439273668, "characteristics.90th_percentile_latency_us": 439273.668, "characteristics.enhancing tumor": 0.7793, "characteristics.mean": 0.85353, "characteristics.tumor core": 0.8688, "characteristics.whole tumor": 0.9125, "ck_system": "AGX_Xavier_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2", "host_memory_capacity": "32GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "25374b28bca49c21", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 436391607, "90.00 percentile latency (ns)": 439273668, "90th percentile latency (ns)": 439273668, "95.00 percentile latency (ns)": 440216998, "97.00 percentile latency (ns)": 440882977, "99.00 percentile latency (ns)": 441936306, "99.90 percentile latency (ns)": 442847339, "Max latency (ns)": 443015405, "Mean latency (ns)": 436642629, "Min duration satisfied": "Yes", "Min latency (ns)": 432910202, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 2.29, "QPS w/o loadgen overhead": 2.29, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA AGX Xavier", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 439.273668, "characteristics.90th_percentile_latency_ns": 439273668.0, "characteristics.90th_percentile_latency_s": 0.439273668, "characteristics.90th_percentile_latency_us": 439273.668, "characteristics.enhancing tumor": 0.7793, "characteristics.mean": 0.85353, "characteristics.tumor core": 0.8688, "characteristics.whole tumor": 0.9125, "ck_system": "AGX_Xavier_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2", "host_memory_capacity": "32GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 8, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32GB", "host_storage_type": "eMMC 5.1", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/AGX_Xavier_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/AGX_Xavier_TRT", "system_name": "NVIDIA Jetson AGX Xavier 32GB (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 2.25225, "task": "image segmentation", "task2": "image segmentation", "total_cores": 8, "uid": "475d9bfc60af4424", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 37025669, "90.00 percentile latency (ns)": 37213311, "90th percentile latency (ns)": 37213311, "95.00 percentile latency (ns)": 37322536, "97.00 percentile latency (ns)": 37649630, "99.00 percentile latency (ns)": 41281292, "99.90 percentile latency (ns)": 114176181, "Max latency (ns)": 127167322, "Mean latency (ns)": 37117879, "Min duration satisfied": "Yes", "Min latency (ns)": 35258194, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 26.93, "QPS w/o loadgen overhead": 26.94, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 37.213311, "characteristics.90th_percentile_latency_ns": 37213311.0, "characteristics.90th_percentile_latency_s": 0.037213311, "characteristics.90th_percentile_latency_us": 37213.311, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.8538, "characteristics.tumor core": 0.8693, "characteristics.whole tumor": 0.9126, "ck_system": "DGX-A100_A100-SXM4x1_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "5bba14739506d3b1", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 37025669, "90.00 percentile latency (ns)": 37213311, "90th percentile latency (ns)": 37213311, "95.00 percentile latency (ns)": 37322536, "97.00 percentile latency (ns)": 37649630, "99.00 percentile latency (ns)": 41281292, "99.90 percentile latency (ns)": 114176181, "Max latency (ns)": 127167322, "Mean latency (ns)": 37117879, "Min duration satisfied": "Yes", "Min latency (ns)": 35258194, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 26.93, "QPS w/o loadgen overhead": 26.94, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 37.213311, "characteristics.90th_percentile_latency_ns": 37213311.0, "characteristics.90th_percentile_latency_s": 0.037213311, "characteristics.90th_percentile_latency_us": 37213.311, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.8538, "characteristics.tumor core": 0.8693, "characteristics.whole tumor": 0.9126, "ck_system": "DGX-A100_A100-SXM4x1_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "0c9a3dd94f65a6e1", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 192783790, "90.00 percentile latency (ns)": 192918583, "90th percentile latency (ns)": 192918583, "95.00 percentile latency (ns)": 192933111, "97.00 percentile latency (ns)": 192949663, "99.00 percentile latency (ns)": 192969568, "99.90 percentile latency (ns)": 193027117, "Max latency (ns)": 193036545, "Mean latency (ns)": 192800087, "Min duration satisfied": "Yes", "Min latency (ns)": 192593681, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 5.19, "QPS w/o loadgen overhead": 5.19, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "5GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4 (1x1g.5gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 192.918583, "characteristics.90th_percentile_latency_ns": 192918583.0, "characteristics.90th_percentile_latency_s": 0.192918583, "characteristics.90th_percentile_latency_us": 192918.583, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9125, "ck_system": "DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "5e52963634e7f98a", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 192783790, "90.00 percentile latency (ns)": 192918583, "90th percentile latency (ns)": 192918583, "95.00 percentile latency (ns)": 192933111, "97.00 percentile latency (ns)": 192949663, "99.00 percentile latency (ns)": 192969568, "99.90 percentile latency (ns)": 193027117, "Max latency (ns)": 193036545, "Mean latency (ns)": 192800087, "Min duration satisfied": "Yes", "Min latency (ns)": 192593681, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 5.19, "QPS w/o loadgen overhead": 5.19, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "5GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-SXM4 (1x1g.5gb MIG)", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 192.918583, "characteristics.90th_percentile_latency_ns": 192918583.0, "characteristics.90th_percentile_latency_s": 0.192918583, "characteristics.90th_percentile_latency_us": 192918.583, "characteristics.enhancing tumor": 0.7795, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.869, "characteristics.whole tumor": 0.9125, "ck_system": "DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "1 TB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "15 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/DGX-A100_A100-SXM4x1-MIG_1x1g.5gb_TRT", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 29.4118, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "841858f3f46d3bb1", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 172767732, "90.00 percentile latency (ns)": 173680417, "90th percentile latency (ns)": 173680417, "95.00 percentile latency (ns)": 173955210, "97.00 percentile latency (ns)": 174406575, "99.00 percentile latency (ns)": 175711884, "99.90 percentile latency (ns)": 176690782, "Max latency (ns)": 266778894, "Mean latency (ns)": 172635563, "Min duration satisfied": "Yes", "Min latency (ns)": 164688597, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 5.79, "QPS w/o loadgen overhead": 5.79, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 173.680417, "characteristics.90th_percentile_latency_ns": 173680417.0, "characteristics.90th_percentile_latency_s": 0.173680417, "characteristics.90th_percentile_latency_us": 173680.417, "characteristics.enhancing tumor": 0.7793, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.8692, "characteristics.whole tumor": 0.9125, "ck_system": "T4x1_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x1_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x1_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.25, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "aa5a4655edde4c08", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 172767732, "90.00 percentile latency (ns)": 173680417, "90th percentile latency (ns)": 173680417, "95.00 percentile latency (ns)": 173955210, "97.00 percentile latency (ns)": 174406575, "99.00 percentile latency (ns)": 175711884, "99.90 percentile latency (ns)": 176690782, "Max latency (ns)": 266778894, "Mean latency (ns)": 172635563, "Min duration satisfied": "Yes", "Min latency (ns)": 164688597, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 5.79, "QPS w/o loadgen overhead": 5.79, "Result is": "VALID", "SUT name": "Triton_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 173.680417, "characteristics.90th_percentile_latency_ns": 173680417.0, "characteristics.90th_percentile_latency_s": 0.173680417, "characteristics.90th_percentile_latency_us": 173680.417, "characteristics.enhancing tumor": 0.7793, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.8692, "characteristics.whole tumor": 0.9125, "ck_system": "T4x1_TRT_Triton", "ck_used": true, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 120, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7V13 64-Core Processor", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x1_TRT_Triton", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.5 LTS (Linux-5.4.0-1055-azure-x86_64-with-Ubuntu-18.04-bionic)", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0, Triton 20.09; GCC 7.5.0; Python 3.7.10", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "Powered by CK v2.5.8 (https://github.com/ctuning/ck)", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x1_TRT_Triton", "system_name": "Microsoft Corporation 7.0 (Virtual Machine)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.25, "task": "image segmentation", "task2": "image segmentation", "total_cores": 240, "uid": "cd79466f4152a85d", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 156716608, "90.00 percentile latency (ns)": 157755172, "90th percentile latency (ns)": 157755172, "95.00 percentile latency (ns)": 157987706, "97.00 percentile latency (ns)": 158098276, "99.00 percentile latency (ns)": 158333423, "99.90 percentile latency (ns)": 159674941, "Max latency (ns)": 267469509, "Mean latency (ns)": 156740931, "Min duration satisfied": "Yes", "Min latency (ns)": 143242348, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 6.38, "QPS w/o loadgen overhead": 6.38, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 157.755172, "characteristics.90th_percentile_latency_ns": 157755172.0, "characteristics.90th_percentile_latency_s": 0.157755172, "characteristics.90th_percentile_latency_us": 157755.172, "characteristics.enhancing tumor": 0.7793, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.8692, "characteristics.whole tumor": 0.9125, "ck_system": "T4x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "3d-unet-99", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x1_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x1_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x T4, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.25, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "791a574c405894c2", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 156716608, "90.00 percentile latency (ns)": 157755172, "90th percentile latency (ns)": 157755172, "95.00 percentile latency (ns)": 157987706, "97.00 percentile latency (ns)": 158098276, "99.00 percentile latency (ns)": 158333423, "99.90 percentile latency (ns)": 159674941, "Max latency (ns)": 267469509, "Mean latency (ns)": 156740931, "Min duration satisfied": "Yes", "Min latency (ns)": 143242348, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 6.38, "QPS w/o loadgen overhead": 6.38, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "16 GB", "accelerator_memory_configuration": "GDDR6", "accelerator_model_name": "NVIDIA T4", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 157.755172, "characteristics.90th_percentile_latency_ns": 157755172.0, "characteristics.90th_percentile_latency_s": 0.157755172, "characteristics.90th_percentile_latency_us": 157755.172, "characteristics.enhancing tumor": 0.7793, "characteristics.mean": 0.85367, "characteristics.tumor core": 0.8692, "characteristics.whole tumor": 0.9125, "ck_system": "T4x1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 28, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8280 CPU @ 2.70GHz", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "ECC off", "informal_model": "3d-unet-99.9", "input_data_types": "int8", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/T4x1_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/T4x1_TRT", "system_name": "Supermicro 4029GP-TRT-OTO-28 (1x T4, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 6.25, "task": "image segmentation", "task2": "image segmentation", "total_cores": 56, "uid": "5ac39c494198ccee", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 34956454, "90.00 percentile latency (ns)": 35284801, "90th percentile latency (ns)": 35284801, "95.00 percentile latency (ns)": 35397661, "97.00 percentile latency (ns)": 35438326, "99.00 percentile latency (ns)": 35488746, "99.90 percentile latency (ns)": 35654907, "Max latency (ns)": 36021693, "Mean latency (ns)": 34974439, "Min duration satisfied": "Yes", "Min latency (ns)": 34419256, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 28.55, "QPS w/o loadgen overhead": 28.59, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 35.284801, "characteristics.90th_percentile_latency_ns": 35284801.0, "characteristics.90th_percentile_latency_s": 0.035284801, "characteristics.90th_percentile_latency_us": 35284.801, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85347, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9125, "ck_system": "A100-PCIex1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT", "system_name": "Gigabyte G482-Z52 (1x A100-PCIe, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "d1fbfd3584461486", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 34956454, "90.00 percentile latency (ns)": 35284801, "90th percentile latency (ns)": 35284801, "95.00 percentile latency (ns)": 35397661, "97.00 percentile latency (ns)": 35438326, "99.00 percentile latency (ns)": 35488746, "99.90 percentile latency (ns)": 35654907, "Max latency (ns)": 36021693, "Mean latency (ns)": 34974439, "Min duration satisfied": "Yes", "Min latency (ns)": 34419256, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 28.55, "QPS w/o loadgen overhead": 28.59, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "40GB", "accelerator_memory_configuration": "HBM2", "accelerator_model_name": "NVIDIA A100-PCIe", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 35.284801, "characteristics.90th_percentile_latency_ns": 35284801.0, "characteristics.90th_percentile_latency_s": 0.035284801, "characteristics.90th_percentile_latency_us": 35284.801, "characteristics.enhancing tumor": 0.779, "characteristics.mean": 0.85347, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9125, "ck_system": "A100-PCIex1_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "TensorRT 7.2, CUDA 11.0 Update 1", "host_memory_capacity": "768 GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 64, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "AMD EPYC 7742", "host_processors_per_node": 2, "host_storage_capacity": "4 TB", "host_storage_type": "NVMe SSD", "hw_notes": "", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/A100-PCIex1_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "TensorRT 7.2, CUDA 11.0 Update 1, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/A100-PCIex1_TRT", "system_name": "Gigabyte G482-Z52 (1x A100-PCIe, TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 23.8095, "task": "image segmentation", "task2": "image segmentation", "total_cores": 128, "uid": "198944ef56fa025e", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 769086004, "90.00 percentile latency (ns)": 771461139, "90th percentile latency (ns)": 771461139, "95.00 percentile latency (ns)": 771906950, "97.00 percentile latency (ns)": 772379568, "99.00 percentile latency (ns)": 778540511, "99.90 percentile latency (ns)": 787793229, "Max latency (ns)": 789757353, "Mean latency (ns)": 768602415, "Min duration satisfied": "Yes", "Min latency (ns)": 758071327, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 1.3, "QPS w/o loadgen overhead": 1.3, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 771.461139, "characteristics.90th_percentile_latency_ns": 771461139.0, "characteristics.90th_percentile_latency_s": 0.771461139, "characteristics.90th_percentile_latency_us": 771461.139, "characteristics.enhancing tumor": 0.7794, "characteristics.mean": 0.85353, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9123, "ck_system": "Xavier_NX_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.0, "formal_model_link": "", "framework": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2", "host_memory_capacity": "8GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/Xavier_NX_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT", "system_name": "NVIDIA Jetson Xavier NX (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "b9a5c9447b4c9da6", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" }, { "50.00 percentile latency (ns)": 769086004, "90.00 percentile latency (ns)": 771461139, "90th percentile latency (ns)": 771461139, "95.00 percentile latency (ns)": 771906950, "97.00 percentile latency (ns)": 772379568, "99.00 percentile latency (ns)": 778540511, "99.90 percentile latency (ns)": 787793229, "Max latency (ns)": 789757353, "Mean latency (ns)": 768602415, "Min duration satisfied": "Yes", "Min latency (ns)": 758071327, "Min queries satisfied": "Yes", "Mode": "Performance", "QPS w/ loadgen overhead": 1.3, "QPS w/o loadgen overhead": 1.3, "Result is": "VALID", "SUT name": "LWIS_Server", "Scenario": "singlestream", "accelerator_frequency": "", "accelerator_host_interconnect": "", "accelerator_interconnect": "", "accelerator_interconnect_topology": "", "accelerator_memory_capacity": "Shared with host", "accelerator_memory_configuration": "SRAM", "accelerator_model_name": "NVIDIA Xavier NX", "accelerator_on-chip_memories": "", "accelerators_per_node": 1, "accuracy_log_probability": 0, "accuracy_log_rng_seed": 0, "accuracy_log_sampling_target": 0, "characteristics.90th_percentile_latency_ms": 771.461139, "characteristics.90th_percentile_latency_ns": 771461139.0, "characteristics.90th_percentile_latency_s": 0.771461139, "characteristics.90th_percentile_latency_us": 771461.139, "characteristics.enhancing tumor": 0.7794, "characteristics.mean": 0.85353, "characteristics.tumor core": 0.8689, "characteristics.whole tumor": 0.9123, "ck_system": "Xavier_NX_TRT", "ck_used": false, "cooling": "", "dataset": "BraTS 2019", "dataset_link": "https://www.med.upenn.edu/cbica/brats2019/data.html", "dim_x_default": "characteristics.90th_percentile_latency_ms", "dim_y_default": "characteristics.mean", "dim_y_maximize": true, "division": "closed", "formal_model": "3d-unet", "formal_model_accuracy": 99.9, "formal_model_link": "", "framework": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2", "host_memory_capacity": "8GB", "host_memory_configuration": "", "host_networking": "", "host_networking_topology": "", "host_processor_caches": "", "host_processor_core_count": 6, "host_processor_frequency": "", "host_processor_interconnect": "", "host_processor_model_name": "NVIDIA Carmel (ARMv8.2)", "host_processors_per_node": 1, "host_storage_capacity": "32GB", "host_storage_type": "Micro SD Card", "hw_notes": "GPU and both DLAs are used in resnet50, ssd-mobilenet, and ssd-resnet34, in Offline and MultiStream scenarios", "informal_model": "3d-unet-99.9", "input_data_types": "fp16", "key.accuracy": "characteristics.mean", "max_async_queries": 1, "max_duration (ms)": 0, "max_query_count": 0, "min_duration (ms)": 60000, "min_query_count": 1024, "mlperf_version": 0.7, "normalize_cores": 1, "normalize_processors": 1, "note_code": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/code", "note_details": "https://github.com/mlcommons/inference_results_v0.7/tree/master/closed/NVIDIA/results/Xavier_NX_TRT", "number_of_nodes": 1, "operating_system": "Ubuntu 18.04.4", "other_software_stack": "20.09 Jetson CUDA-X AI Developer Preview, TensorRT 7.2, CUDA 10.2, cuDNN 8.0.2, DALI 0.25.0", "performance_issue_same": true, "performance_issue_same_index": 0, "performance_issue_unique": true, "performance_sample_count": 16, "print_timestamps": true, "problem": false, "qsl_rng_seed": 12786827339337101903, "retraining": "N", "sample_index_rng_seed": 12640797754436136668, "samples_per_query": 1, "schedule_rng_seed": 3135815929913719677, "starting_weights_filename": "224_224_160_dyanmic_bs.onnx", "status": "available", "submitter": "NVIDIA", "submitter_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.submitter/NVIDIA", "sw_notes": "", "system_link": "https://github.com/ctuning/ck-mlperf-inference/tree/main/bench.mlperf.system/Xavier_NX_TRT", "system_name": "NVIDIA Jetson Xavier NX (TensorRT)", "system_type": "edge", "target_latency (ns)": 0, "target_qps": 1.12613, "task": "image segmentation", "task2": "image segmentation", "total_cores": 6, "uid": "86a6c4a4cbf78a82", "use_accelerator": true, "weight_data_types": "int8", "weight_transformations": "quantization, affine fusion" } ]